preliminary implementation of new commit

(cherry picked from commit 1b2a0fd1ae1e6dfc18e4f75f73cd7befb47cf538)

7 files changed, 557 insertions, 169 deletions

diff --git a/src/cstore/cstore-varref.cpp b/src/cstore/cstore-varref.cpp
index 1e3be90..7549834 100644
--- a/src/cstore/cstore-varref.cpp
+++ b/src/cstore/cstore-varref.cpp
@@ -122,14 +122,12 @@ Cstore::VarRef::process_ref(const Cpath& ref_comps,
         return;
       }
     }
-    if (pcomps.size() < _orig_path_comps.size()) {
-      // within the original path. @ translates to the path comp.
-      pcomps.push(_orig_path_comps[pcomps.size()]);
-      process_ref(rcomps, pcomps, def->getType(1));
-      return;
-    }
-    if (def->isValue() || def->isTag()) {
-      // invalid ref
+    if (!def->isSingleLeafNode() && !def->isMultiLeafNode()) {
+      if (pcomps.size() < _orig_path_comps.size()) {
+        // within the original path. @ translates to the path comp.
+        pcomps.push(_orig_path_comps[pcomps.size()]);
+        process_ref(rcomps, pcomps, def->getType(1));
+      }
       return;
     }
     // handle leaf node
@@ -186,12 +184,16 @@ Cstore::VarRef::process_ref(const Cpath& ref_comps,
     // just text. go down 1 level.
     if (got_tmpl && def->isTagNode()) {
       // at "tag node". need to go down 1 more level.
-      if (pcomps.size() >= _orig_path_comps.size()) {
+      if (pcomps.size() > _orig_path_comps.size()) {
         // already under the original node. invalid ref.
         return;
+      } else if (pcomps.size() == _orig_path_comps.size()) {
+        // at the tag value. use the at_string.
+        pcomps.push(_at_string);
+      } else  {
+        // within the original path. take the original tag value.
+        pcomps.push(_orig_path_comps[pcomps.size()]);
       }
-      // within the original path. take the original tag value.
-      pcomps.push(_orig_path_comps[pcomps.size()]);
     }
     pcomps.push(cr_comp);
     process_ref(rcomps, pcomps, ERROR_TYPE);
@@ -218,11 +220,16 @@ Cstore::VarRef::process_ref(const Cpath& ref_comps,
     } else {
       // single-value node
       string val;
+      vtw_type_e t = def->getType(1);
       if (!_cstore->cfgPathGetValue(pcomps, val, _active)) {
-        return;
+        /* can't get value => treat it as non-existent (empty value
+         * and type ERROR_TYPE)
+         */
+        val = "";
+        t = ERROR_TYPE;
       }
       pcomps.push(val);
-      _paths.push_back(pair<Cpath, vtw_type_e>(pcomps, def->getType(1)));
+      _paths.push_back(pair<Cpath, vtw_type_e>(pcomps, t));
       // at leaf. stop recursion.
     }
   }
@@ -288,6 +295,13 @@ Cstore::VarRef::getSetPath(Cpath& path_comps)
     return false;
   }
   path_comps = _paths[0].first;
+  /* note that for "varref set" operation, the varref must refer to the
+   * "value" of a single-value leaf node, e.g.,
+   * "$VAR(plaintext-password/@)". so pop the last comp to give the
+   * correct path for "set". the caller is responsible for verifying
+   * whether the path points to a single-value leaf node.
+   */
+  path_comps.pop();
   return true;
 }
 
diff --git a/src/cstore/cstore.cpp b/src/cstore/cstore.cpp
index 143e501..b549649 100644
--- a/src/cstore/cstore.cpp
+++ b/src/cstore/cstore.cpp
@@ -36,9 +36,12 @@
 #include <cnode/cnode.hpp>
 #include <cnode/cnode-algorithm.hpp>
 #include <cparse/cparse.hpp>
+#include <commit/commit-algorithm.hpp>
 
 namespace cstore { // begin namespace cstore
 
+using namespace cnode;
+
 ////// constants
 //// node status
 const string Cstore::C_NODE_STATUS_DELETED = "deleted";
@@ -707,10 +710,10 @@ Cstore::getCompletionEnv(const Cpath& comps, string& env)
        * shell into an array of values.
        */
       free(buf);
-    } else if (def->getActions(syntax_act)->vtw_list_head) {
+    } else if (def->getActions(syntax_act)) {
       // look for "self ref in values" from syntax
-      const valstruct *vals = get_syntax_self_in_valstruct(
-                                def->getActions(syntax_act)->vtw_list_head);
+      const valstruct *vals
+        = get_syntax_self_in_valstruct(def->getActions(syntax_act));
       if (vals) {
         if (vals->cnt == 0 && vals->val) {
           comp_vals.push_back(vals->val);
@@ -1534,21 +1537,8 @@ Cstore::cfgPathEffective(const Cpath& path_comps)
   }
 
   bool in_work = cfg_path_exists(path_comps, false, false);
-  if (in_active && in_work) {
-    // case (1)
-    return true;
-  }
-
-  auto_ptr<SavePaths> save(create_save_paths());
-  append_cfg_path(path_comps);
-  if (!in_active && in_work) {
-    // check if case (2)
-    return marked_committed(def, true);
-  } else if (in_active && !in_work) {
-    // check if case (3)
-    return !marked_committed(def, false);
-  }
-  return false;
+  return commit::isCommitPathEffective(*this, path_comps, def,
+                                       in_active, in_work);
 }
 
 /* get names of "effective" child nodes of specified path during commit
@@ -1797,7 +1787,7 @@ Cstore::loadFile(const char *filename)
   }
 
   // get the config tree from the file
-  cnode::CfgNode *froot = cparse::parse_file(fin, *this);
+  CfgNode *froot = cparse::parse_file(fin, *this);
   if (!froot) {
     output_user("Failed to parse specified config file\n");
     return false;
@@ -1805,13 +1795,13 @@ Cstore::loadFile(const char *filename)
 
   // get the config tree from the active config
   Cpath args;
-  cnode::CfgNode aroot(*this, args, true, true);
+  CfgNode aroot(*this, args, true, true);
 
   // get the "commands diff" between the two
   vector<Cpath> del_list;
   vector<Cpath> set_list;
   vector<Cpath> com_list;
-  cnode::get_cmds_diff(aroot, *froot, del_list, set_list, com_list);
+  get_cmds_diff(aroot, *froot, del_list, set_list, com_list);
 
   // "apply" the changes to the working config
   for (size_t i = 0; i < del_list.size(); i++) {
@@ -1890,6 +1880,45 @@ Cstore::unmarkCfgPathChanged(const Cpath& path_comps)
   return unmark_changed_with_descendants();
 }
 
+// execute the specified actions
+bool
+Cstore::executeTmplActions(char *at_str, const Cpath& path,
+                           const Cpath& disp_path, const vtw_node *actions,
+                           const vtw_def *def)
+{
+  string sdisp = " ";
+  sdisp += disp_path.to_string();
+  sdisp += " ";
+  set_at_string(at_str);
+
+  auto_ptr<SavePaths> save(create_save_paths());
+  append_cfg_path(path);
+  append_tmpl_path(path);
+
+  var_ref_handle = (void *) this;
+  // const_cast for legacy code
+  bool ret = execute_list(const_cast<vtw_node *>(actions), def,
+                          sdisp.c_str(), false);
+  var_ref_handle = NULL;
+  return ret;
+}
+
+bool
+Cstore::cfgPathMarkedCommitted(const Cpath& path_comps, bool is_delete)
+{
+  auto_ptr<SavePaths> save(create_save_paths());
+  append_cfg_path(path_comps);
+  return marked_committed(is_delete);
+}
+
+bool
+Cstore::markCfgPathCommitted(const Cpath& path_comps, bool is_delete)
+{
+  auto_ptr<SavePaths> save(create_save_paths());
+  append_cfg_path(path_comps);
+  return mark_committed(is_delete);
+}
+
 
 ////// protected functions
 Cstore::SavePaths::~SavePaths() {
@@ -2665,9 +2694,15 @@ Cstore::validate_val(const tr1::shared_ptr<Ctemplate>& def, const char *value)
   }
 
   // validate_value() may change "value". make a copy first.
-  char *vbuf = strdup(value);
-  bool ret = validate_val_impl(def, vbuf);
-  free(vbuf);
+  auto_ptr<char> vbuf(strdup(value));
+
+  /* set the handle to be used during validate_value() for var ref
+   * processing. this is a global var in cli_new.c.
+   */
+  var_ref_handle = (void *) this;
+  bool ret = validate_value(def->getDef(), vbuf.get());
+  var_ref_handle = NULL;
+
   return ret;
 }
 
diff --git a/src/cstore/cstore.hpp b/src/cstore/cstore.hpp
index f2163a8..af4a56d 100644
--- a/src/cstore/cstore.hpp
+++ b/src/cstore/cstore.hpp
@@ -40,6 +40,13 @@ extern "C" void* Perl_get_context(void)
                             "calling %s() without config session", \
                             __func__);
 
+// forward decl
+namespace cnode {
+class CfgNode;
+}
+namespace commit {
+class PrioNode;
+}
 
 namespace cstore { // begin namespace cstore
 
@@ -163,6 +170,13 @@ public:
   virtual bool inSession() = 0;
   // commit
   bool unmarkCfgPathChanged(const Cpath& path_comps);
+  bool executeTmplActions(char *at_str, const Cpath& path,
+                          const Cpath& disp_path, const vtw_node *actions,
+                          const vtw_def *def);
+  bool cfgPathMarkedCommitted(const Cpath& path_comps, bool is_delete);
+  bool markCfgPathCommitted(const Cpath& path_comps, bool is_delete);
+  virtual bool clearCommittedMarkers() = 0;
+  virtual bool commitConfig(commit::PrioNode& pnode) = 0;
   // load
   bool loadFile(const char *filename);
 
@@ -361,14 +375,6 @@ private:
   virtual bool get_comment(string& comment, bool active_cfg) = 0;
   virtual bool marked_display_default(bool active_cfg) = 0;
 
-  // observers during commit operation
-  virtual bool marked_committed(const tr1::shared_ptr<Ctemplate>& def,
-                                bool is_set) = 0;
-
-  // these operate on both current tmpl and work paths
-  virtual bool validate_val_impl(const tr1::shared_ptr<Ctemplate>& def,
-                                 char *value) = 0;
-
   // observers for "edit/tmpl levels" (for "edit"-related operations)
   /* note that these should be handled in the base class since they
    * should not be implementation-specific. however, current definitions
@@ -385,6 +391,10 @@ private:
   virtual void get_edit_level(Cpath& path_comps) = 0;
   virtual bool edit_level_at_root() = 0;
 
+  // functions for commit operation
+  virtual bool marked_committed(bool is_delete) = 0;
+  virtual bool mark_committed(bool is_delete) = 0;
+
   // these are for testing/debugging
   virtual string cfg_path_to_str() = 0;
   virtual string tmpl_path_to_str() = 0;
diff --git a/src/cstore/ctemplate.hpp b/src/cstore/ctemplate.hpp
index ebe4a5b..96c47b5 100644
--- a/src/cstore/ctemplate.hpp
+++ b/src/cstore/ctemplate.hpp
@@ -20,15 +20,13 @@
 #include <tr1/memory>
 
 #include <cli_cstore.h>
-#include <cstore/cstore.hpp>
 
 namespace cstore { // begin namespace cstore
 
-using namespace std;
-
 class Ctemplate {
 public:
-  Ctemplate(tr1::shared_ptr<vtw_def> def) : _def(def), _is_value(false) {};
+  Ctemplate(std::tr1::shared_ptr<vtw_def> def)
+    : _def(def), _is_value(false) {};
   ~Ctemplate() {};
 
   bool isValue() const { return _is_value; };
@@ -82,8 +80,8 @@ public:
   const char *getNodeHelp() const { return _def->def_node_help; };
   const char *getEnumeration() const { return _def->def_enumeration; };
   const char *getAllowed() const { return _def->def_allowed; };
-  const vtw_list *getActions(vtw_act_type act) const {
-    return &(_def->actions[act]);
+  const vtw_node *getActions(vtw_act_type act) const {
+    return _def->actions[act].vtw_list_head;
   };
   const char *getCompHelp() const { return _def->def_comp_help; };
   const char *getValHelp() const { return _def->def_val_help; };
@@ -92,6 +90,13 @@ public:
   unsigned int getPriority() const { return _def->def_priority; };
 
   void setIsValue(bool is_val) { _is_value = is_val; };
+  void setPriority(unsigned int p) const {
+    /* this changes the parsed template and is only used during commit IF the
+     * priority specified in the template violates the "hierarchical
+     * constraint" and therefore needs to be changed.
+     */
+    _def->def_priority = p;
+  }
 
   const vtw_def *getDef() const {
     /* XXX this is a hack for code that has not been converted and is still
@@ -118,7 +123,7 @@ private:
    *     suitable containers so that memory allocation/deallocation can be
    *     handled properly.
    */
-  tr1::shared_ptr<vtw_def> _def;
+  std::tr1::shared_ptr<vtw_def> _def;
   bool _is_value; /* whether the last path component is a "value". set by
                    * the cstore in get_parsed_tmpl().
                    */
diff --git a/src/cstore/unionfs/cstore-unionfs.cpp b/src/cstore/unionfs/cstore-unionfs.cpp
index e1a7dfd..659a985 100644
--- a/src/cstore/unionfs/cstore-unionfs.cpp
+++ b/src/cstore/unionfs/cstore-unionfs.cpp
@@ -25,6 +25,8 @@
 
 #include <cli_cstore.h>
 #include <cstore/unionfs/cstore-unionfs.hpp>
+#include <cnode/cnode.hpp>
+#include <commit/commit-algorithm.hpp>
 
 namespace cstore { // begin namespace cstore
 namespace unionfs { // begin namespace unionfs
@@ -56,7 +58,7 @@ const string UnionfsCstore::C_MARKER_DEF_VALUE  = "def";
 const string UnionfsCstore::C_MARKER_DEACTIVATE = ".disable";
 const string UnionfsCstore::C_MARKER_CHANGED = ".modified";
 const string UnionfsCstore::C_MARKER_UNSAVED = ".unsaved";
-const string UnionfsCstore::C_COMMITTED_MARKER_FILE = "/tmp/.changes";
+const string UnionfsCstore::C_COMMITTED_MARKER_FILE = ".changes";
 const string UnionfsCstore::C_COMMENT_FILE = ".comment";
 const string UnionfsCstore::C_TAG_NAME = "node.tag";
 const string UnionfsCstore::C_VAL_NAME = "node.val";
@@ -181,6 +183,7 @@ UnionfsCstore::UnionfsCstore(bool use_edit_level)
   }
   if ((val = getenv(C_ENV_TMP_ROOT.c_str()))) {
     tmp_root = val;
+    init_commit_data();
   }
   if ((val = getenv(C_ENV_ACTIVE_ROOT.c_str()))) {
     active_root = val;
@@ -240,6 +243,7 @@ UnionfsCstore::UnionfsCstore(const string& sid, string& env)
   work_root = (C_DEF_WORK_PREFIX + sid);
   change_root = (C_DEF_CHANGE_PREFIX + sid);
   tmp_root = (C_DEF_TMP_PREFIX + sid);
+  init_commit_data();
 
   string declr = " declare -x -r "; // readonly vars
   env += " umask 002; {";
@@ -345,15 +349,7 @@ UnionfsCstore::setupSession()
     }
 
     // union mount
-    string mopts = "dirs=";
-    mopts += change_root.path_cstr();
-    mopts += "=rw:";
-    mopts += active_root.path_cstr();
-    mopts += "=ro";
-    if (mount("unionfs", work_root.path_cstr(), "unionfs", 0,
-              mopts.c_str()) != 0) {
-      output_internal("setup session mount failed [%s][%s]\n",
-                      strerror(errno), work_root.path_cstr());
+    if (!do_mount(change_root, active_root, work_root)) {
       return false;
     }
   } else if (!path_is_directory(work_root)) {
@@ -381,9 +377,7 @@ UnionfsCstore::teardownSession()
   }
 
   // unmount the work root (union)
-  if (umount(wstr.c_str()) != 0) {
-    output_internal("teardown session umount failed [%s][%s]\n",
-                    strerror(errno), wstr.c_str());
+  if (!do_umount(work_root)) {
     return false;
   }
 
@@ -415,6 +409,275 @@ UnionfsCstore::inSession()
           && path_exists(work_root) && path_is_directory(work_root));
 }
 
+bool
+UnionfsCstore::clearCommittedMarkers()
+{
+  try {
+    b_fs::remove(commit_marker_file.path_cstr());
+  } catch (...) {
+    output_internal("failed to clear committed markers\n");
+    return false;
+  }
+  return true;
+}
+
+bool
+UnionfsCstore::construct_commit_active(commit::PrioNode& node)
+{
+  auto_ptr<SavePaths> save(create_save_paths());
+  reset_paths();
+  append_cfg_path(node.getCommitPath());
+
+  FsPath ap(get_active_path());
+  FsPath wp(get_work_path());
+  FsPath tap(tmp_active_root);
+  tap /= mutable_cfg_path;
+
+  if (path_exists(tap)) {
+    output_internal("rm[%s]\n", tap.path_cstr());
+    if (b_fs::remove_all(tap.path_cstr()) < 1) {
+      output_user("FAILED\n");
+      return false;
+    }
+    cnode::CfgNode *c = node.getCfgNode();
+    if (c && c->isTag()) {
+      FsPath p(tap);
+      p.pop();
+      if (is_directory_empty(p)) {
+        output_internal("rm[%s]\n", p.path_cstr());
+        if (b_fs::remove_all(p.path_cstr()) < 1) {
+          output_user("FAILED\n");
+          return false;
+        }
+      }
+    }
+  } else {
+    output_internal("no tap[%s]\n", tap.path_cstr());
+  }
+  if (node.succeeded()) {
+    // prio subtree succeeded
+    if (path_exists(wp)) {
+      output_internal("cp[%s]->[%s]\n", wp.path_cstr(), tap.path_cstr());
+      try {
+        recursive_copy_dir(wp, tap, true);
+      } catch (...) {
+        output_user("FAILED\n");
+        return false;
+      }
+    } else {
+      output_internal("no wp[%s]\n", wp.path_cstr());
+    }
+    if (!node.hasSubtreeFailure()) {
+      // whole subtree succeeded => stop recursion
+      return true;
+    }
+    // failure present in subtree
+  } else {
+    // prio subtree failed
+    if (path_exists(ap)) {
+      output_internal("cp[%s]->[%s]\n", ap.path_cstr(), tap.path_cstr());
+      try {
+        recursive_copy_dir(ap, tap, false);
+      } catch (...) {
+        output_user("FAILED\n");
+        return false;
+      }
+    } else {
+      output_internal("no ap[%s]\n", ap.path_cstr());
+    }
+    if (!node.hasSubtreeSuccess()) {
+      // whole subtree failed => stop recursion
+      return true;
+    }
+    // success present in subtree
+  }
+  for (size_t i = 0; i < node.numChildNodes(); i++) {
+    if (!construct_commit_active(*(node.childAt(i)))) {
+      return false;
+    }
+  }
+  return true;
+}
+
+bool
+UnionfsCstore::mark_dir_changed(const FsPath& d, const FsPath& root)
+{
+  if (!path_is_directory(d)) {
+    output_internal("mark_dir_changed on non-directory [%s]\n",
+                    d.path_cstr());
+    return false;
+  }
+
+  FsPath marker(d);
+  while (marker.size() >= root.size()) {
+    marker.push(C_MARKER_CHANGED);
+    if (path_exists(marker)) {
+      // reached a node already marked => done
+      break;
+    }
+    if (!create_file(marker)) {
+      output_internal("failed to mark changed [%s]\n", marker.path_cstr());
+      return false;
+    }
+    marker.pop();
+    marker.pop();
+  }
+  return true;
+}
+
+bool
+UnionfsCstore::sync_dir(const FsPath& src, const FsPath& dst,
+                        const FsPath& root)
+{
+  if (!path_exists(src) || !path_exists(dst)) {
+    output_user("sync_dir with non-existing dir(s)[%s][%s]\n",
+                src.path_cstr(), dst.path_cstr());
+    return false;
+  }
+  MapT<string, bool> smap;
+  MapT<string, bool> dmap;
+  vector<string> sentries;
+  vector<string> dentries;
+  check_dir_entries(src, &sentries, false);
+  check_dir_entries(dst, &dentries, false);
+  for (size_t i = 0; i < sentries.size(); i++) {
+    smap[sentries[i]] = true;
+  }
+  for (size_t i = 0; i < dentries.size(); i++) {
+    dmap[dentries[i]] = true;
+    if (smap.find(dentries[i]) == smap.end()) {
+      // entry in dst but not in src => delete
+      FsPath d(dst);
+      if (!mark_dir_changed(d, root)) {
+        return false;
+      }
+      push_path(d, dentries[i].c_str());
+      if (b_fs::remove_all(d.path_cstr()) < 1) {
+        return false;
+      }
+    } else {
+      // entry in both src and dst
+      FsPath s(src);
+      FsPath d(dst);
+      push_path(s, dentries[i].c_str());
+      push_path(d, dentries[i].c_str());
+      if (path_is_regular(s) && path_is_regular(d)) {
+        // it's file => compare and replace if necessary
+        string ds, dd;
+        if (!read_whole_file(s, ds) || !read_whole_file(d, dd)) {
+          // error
+          output_user("failed to replace file [%s][%s]\n",
+                      s.path_cstr(), d.path_cstr());
+          return false;
+        }
+        if (ds != dd) {
+          // need to replace
+          if (!write_file(d, ds)) {
+            output_user("failed to write file [%s]\n", d.path_cstr());
+            return false;
+          }
+          d.pop();
+          if (!mark_dir_changed(d, root)) {
+            return false;
+          }
+        }
+      } else if (path_is_directory(s) && path_is_directory(d)) {
+        // it's dir => recurse
+        if (!sync_dir(s, d, root)) {
+          return false;
+        }
+      } else {
+        // something is wrong
+        output_user("inconsistent config entry [%s][%s]\n",
+                    s.path_cstr(), d.path_cstr());
+        return false;
+      }
+    }
+  }
+  for (size_t i = 0; i < sentries.size(); i++) {
+    if (dmap.find(sentries[i]) == dmap.end()) {
+      // entry in src but not in dst => copy
+      FsPath s(src);
+      FsPath d(dst);
+      push_path(s, sentries[i].c_str());
+      push_path(d, sentries[i].c_str());
+      try {
+        if (path_is_regular(s)) {
+          // it's file
+          b_fs::copy_file(s.path_cstr(), d.path_cstr());
+        } else {
+          // dir
+          recursive_copy_dir(s, d, true);
+        }
+        d.pop();
+        if (!mark_dir_changed(d, root)) {
+          return false;
+        }
+      } catch (...) {
+        output_user("copy failed [%s][%s]\n", s.path_cstr(), d.path_cstr());
+        return false;
+      }
+    }
+  }
+
+  return true;
+}
+
+bool
+UnionfsCstore::commitConfig(commit::PrioNode& node)
+{
+  // make a copy of current "work" dir
+  try {
+    if (path_exists(tmp_work_root)) {
+      output_internal("rm[%s]\n", tmp_work_root.path_cstr());
+      if (b_fs::remove_all(tmp_work_root.path_cstr()) < 1) {
+        output_user("FAILED\n");
+        return false;
+      }
+    }
+    output_internal("cp[%s]->[%s]\n", work_root.path_cstr(),
+                    tmp_work_root.path_cstr());
+    recursive_copy_dir(work_root, tmp_work_root, true);
+  } catch (...) {
+    output_user("FAILED\n");
+    return false;
+  }
+
+  if (!construct_commit_active(node)) {
+    return false;
+  }
+
+  if (!do_umount(work_root)) {
+    return false;
+  }
+  if (b_fs::remove_all(change_root.path_cstr()) < 1
+      || b_fs::remove_all(active_root.path_cstr()) < 1) {
+    output_user("failed to remove existing directories\n");
+    return false;
+  }
+  try {
+    b_fs::create_directories(change_root.path_cstr());
+    recursive_copy_dir(tmp_active_root, active_root, true);
+  } catch (...) {
+    return false;
+  }
+  if (!do_mount(change_root, active_root, work_root)) {
+    return false;
+  }
+  if (!sync_dir(tmp_work_root, work_root, work_root)) {
+    return false;
+  }
+#if 0
+  if (b_fs::remove_all(tmp_work_root.path_cstr()) < 1
+      || b_fs::remove_all(tmp_active_root.path_cstr()) < 1) {
+    output_user("failed to remove temp directories\n");
+    return false;
+  }
+#endif
+  // all done
+  return true;
+}
+
 
 ////// virtual functions defined in base class
 /* check if current tmpl_path is a valid tmpl dir.
@@ -932,58 +1195,6 @@ UnionfsCstore::cfg_node_changed()
   return path_exists(marker);
 }
 
-/* XXX currently "committed marking" is done inside commit.
- *     TODO move "committed marking" out of commit and into low-level
- *     implementation (here).
- */
-/* return whether current "cfg path" has been committed, i.e., whether
- * the set or delete operation on the path has been processed by commit.
- *   is_set: whether the operation is set (for sanity check as there can
- *           be only one operation on the path).
- */
-bool
-UnionfsCstore::marked_committed(const tr1::shared_ptr<Ctemplate>& def,
-                                bool is_set)
-{
-  FsPath cpath = mutable_cfg_path;
-  string com_str = cpath.path_cstr();
-  com_str += "/";
-  if (def->isLeafValue()) {
-    // path includes leaf value. construct the right string.
-    string val;
-    cpath.pop(val);
-    val = _unescape_path_name(val);
-    /* XXX current commit implementation escapes value strings for
-     *     single-value nodes but not for multi-value nodes for some
-     *     reason. the following match current behavior.
-     */
-    if (!def->isMulti()) {
-      val = _escape_path_name(val);
-    }
-    com_str = cpath.path_cstr();
-    com_str += "/value:";
-    com_str += val;
-  }
-  com_str = (is_set ? "+ " : "- ") + com_str;
-  return committed_marker_exists(com_str);
-}
-
-bool
-UnionfsCstore::validate_val_impl(const tr1::shared_ptr<Ctemplate>& def,
-                                 char *value)
-{
-  /* XXX filesystem paths/accesses are completely embedded in var ref lib.
-   *     for now, treat the lib as a unionfs-specific implementation.
-   *     generalizing it will need a rewrite.
-   * set the handle to be used during validate_value() for var ref
-   * processing. this is a global var in cli_new.c.
-   */
-  var_ref_handle = (void *) this;
-  bool ret = validate_value(def->getDef(), value);
-  var_ref_handle = NULL;
-  return ret;
-}
-
 void
 UnionfsCstore::get_edit_level(Cpath& pcomps) {
   FsPath opath = mutable_cfg_path; // use a copy
@@ -999,6 +1210,23 @@ UnionfsCstore::get_edit_level(Cpath& pcomps) {
   }
 }
 
+bool
+UnionfsCstore::marked_committed(bool is_delete)
+{
+  string marker;
+  get_committed_marker(is_delete, marker);
+  return find_line_in_file(commit_marker_file, marker);
+}
+
+bool
+UnionfsCstore::mark_committed(bool is_delete)
+{
+  string marker;
+  get_committed_marker(is_delete, marker);
+  // write one marker per line
+  return write_file(commit_marker_file, marker + "\n", true);
+}
+
 string
 UnionfsCstore::cfg_path_to_str() {
   string cpath = mutable_cfg_path.path_cstr();
@@ -1041,36 +1269,48 @@ UnionfsCstore::pop_path(FsPath& path, string& last)
   last = _unescape_path_name(last);
 }
 
-void
-UnionfsCstore::get_all_child_dir_names(const FsPath& root,
-                                       vector<string>& nodes)
+bool
+UnionfsCstore::check_dir_entries(const FsPath& root, vector<string> *cnodes,
+                                 bool filter_nodes, bool empty_check)
 {
   if (!path_exists(root) || !path_is_directory(root)) {
-    // not a valid root. nop.
-    return;
+    // not a valid root => treat as empty
+    return false;
   }
+  bool found = false;
   try {
     b_fs::directory_iterator di(root.path_cstr());
     for (; di != b_fs::directory_iterator(); ++di) {
-      // must be directory
-      if (!path_is_directory(di->path().file_string().c_str())) {
-        continue;
-      }
-      // name cannot start with "."
       string cname = di->path().filename();
-      if (cname.length() < 1 || cname[0] == '.') {
-        continue;
+      if (filter_nodes) {
+        // must be directory
+        if (!path_is_directory(di->path().file_string().c_str())) {
+          continue;
+        }
+        // name cannot start with "."
+        if (cname.length() < 1 || cname[0] == '.') {
+          continue;
+        }
       }
       // found one
-      nodes.push_back(_unescape_path_name(cname));
+      if (empty_check) {
+        // only checking and directory is not empty
+        return true;
+      }
+      if (cnodes) {
+        cnodes->push_back(_unescape_path_name(cname));
+      } else {
+        found = true;
+      }
     }
   } catch (...) {
-    return;
+    // skip the rest
   }
+  return (cnodes ? (cnodes->size() > 0) : found);
 }
 
 bool
-UnionfsCstore::write_file(const FsPath& file, const string& data)
+UnionfsCstore::write_file(const char *file, const string& data, bool append)
 {
   if (data.size() > C_UNIONFS_MAX_FILE_SIZE) {
     output_internal("write_file too large\n");
@@ -1085,7 +1325,10 @@ UnionfsCstore::write_file(const FsPath& file, const string& data)
     // write the file
     ofstream fout;
     fout.exceptions(ofstream::failbit | ofstream::badbit);
-    fout.open(file.path_cstr(), ios_base::out | ios_base::trunc);
+    ios_base::openmode mflags = ios_base::out;
+    mflags |= ((!append || !path_exists(file))
+               ? ios_base::trunc : ios_base::app); // truncate or append
+    fout.open(file, mflags);
     fout << data;
     fout.close();
   } catch (...) {
@@ -1127,19 +1370,54 @@ UnionfsCstore::read_whole_file(const FsPath& fpath, string& data)
   return true;
 }
 
-/* return whether specified "commited marker" exists in the
- * "committed marker file".
+/* recursively copy source directory to destination.
+ * will throw exception (from b_fs) if fail.
  */
+void
+UnionfsCstore::recursive_copy_dir(const FsPath& src, const FsPath& dst,
+                                  bool filter_dot_entries)
+{
+  string src_str = src.path_cstr();
+  string dst_str = dst.path_cstr();
+  b_fs::create_directories(dst.path_cstr());
+
+  b_fs::recursive_directory_iterator di(src_str);
+  for (; di != b_fs::recursive_directory_iterator(); ++di) {
+    const char *oname = di->path().file_string().c_str();
+    string nname = oname;
+    nname.replace(0, src_str.length(), dst_str);
+    if (path_is_directory(oname)) {
+      b_fs::create_directory(nname);
+    } else {
+      if (filter_dot_entries) {
+        string of = di->path().filename();
+        if (!of.empty() && of.at(0) == '.') {
+          // filter dot files
+          continue;
+        }
+      }
+      b_fs::copy_file(di->path(), nname);
+    }
+  }
+}
+
+void
+UnionfsCstore::get_committed_marker(bool is_delete, string& marker)
+{
+  marker = (is_delete ? "-" : "");
+  marker += mutable_cfg_path.path_cstr();
+}
+
 bool
-UnionfsCstore::committed_marker_exists(const string& marker)
+UnionfsCstore::find_line_in_file(const FsPath& file, const string& line)
 {
   bool ret = false;
   try {
-    ifstream fin(C_COMMITTED_MARKER_FILE.c_str());
+    ifstream fin(file.path_cstr());
     while (!fin.eof() && !fin.bad() && !fin.fail()) {
-      string line;
-      getline(fin, line);
-      if (line == marker) {
+      string in;
+      getline(fin, in);
+      if (in == line) {
         ret = true;
         break;
       }
@@ -1151,27 +1429,32 @@ UnionfsCstore::committed_marker_exists(const string& marker)
   return ret;
 }
 
-/* recursively copy source directory to destination.
- * will throw exception (from b_fs) if fail.
- */
-void
-UnionfsCstore::recursive_copy_dir(const FsPath& src, const FsPath& dst)
+bool
+UnionfsCstore::do_mount(const FsPath& rwdir, const FsPath& rdir,
+                        const FsPath& mdir)
 {
-  string src_str = src.path_cstr();
-  string dst_str = dst.path_cstr();
-  b_fs::create_directory(dst.path_cstr());
+  string mopts = "dirs=";
+  mopts += rwdir.path_cstr();
+  mopts += "=rw:";
+  mopts += rdir.path_cstr();
+  mopts += "=ro";
+  if (mount("unionfs", mdir.path_cstr(), "unionfs", 0, mopts.c_str()) != 0) {
+    output_internal("union mount failed [%s][%s]\n",
+                    strerror(errno), mdir.path_cstr());
+    return false;
+  }
+  return true;
+}
 
-  b_fs::recursive_directory_iterator di(src_str);
-  for (; di != b_fs::recursive_directory_iterator(); ++di) {
-    const char *oname = di->path().file_string().c_str();
-    string nname = oname;
-    nname.replace(0, src_str.length(), dst_str);
-    if (path_is_directory(oname)) {
-      b_fs::create_directory(nname);
-    } else {
-      b_fs::copy_file(oname, nname);
-    }
+bool
+UnionfsCstore::do_umount(const FsPath& mdir)
+{
+  if (umount(mdir.path_cstr()) != 0) {
+    output_internal("union umount failed [%s][%s]\n",
+                    strerror(errno), mdir.path_cstr());
+    return false;
   }
+  return true;
 }
 
 bool
diff --git a/src/cstore/unionfs/cstore-unionfs.hpp b/src/cstore/unionfs/cstore-unionfs.hpp
index 7c73225..8113d00 100644
--- a/src/cstore/unionfs/cstore-unionfs.hpp
+++ b/src/cstore/unionfs/cstore-unionfs.hpp
@@ -30,6 +30,11 @@
 #include <cstore/cstore.hpp>
 #include <cstore/unionfs/fspath.hpp>
 
+// forward decl
+namespace commit {
+class PrioNode;
+}
+
 namespace cstore { // begin namespace cstore
 namespace unionfs { // begin namespace unionfs
 
@@ -50,6 +55,8 @@ public:
   bool setupSession();
   bool teardownSession();
   bool inSession();
+  bool clearCommittedMarkers();
+  bool commitConfig(commit::PrioNode& pnode);
 
 private:
   // constants
@@ -85,7 +92,7 @@ private:
   FsPath work_root;   // working root (union)
   FsPath active_root; // active root (readonly part of union)
   FsPath change_root; // change root (r/w part of union)
-  FsPath tmp_root;   // temp root
+  FsPath tmp_root;    // temp root
   FsPath tmpl_root;   // template root
 
   // path buffers
@@ -94,6 +101,22 @@ private:
   FsPath orig_mutable_cfg_path;  // original mutable cfg path
   FsPath orig_tmpl_path;         // original template path
 
+  // for commit processing
+  FsPath tmp_active_root;
+  FsPath tmp_work_root;
+  FsPath commit_marker_file;
+  void init_commit_data() {
+    tmp_active_root = tmp_root;
+    tmp_work_root = tmp_root;
+    commit_marker_file = tmp_root;
+    tmp_active_root.push("active");
+    tmp_work_root.push("work");
+    commit_marker_file.push(C_COMMITTED_MARKER_FILE);
+  }
+  bool construct_commit_active(commit::PrioNode& node);
+  bool mark_dir_changed(const FsPath& d, const FsPath& root);
+  bool sync_dir(const FsPath& src, const FsPath& dst, const FsPath& root);
+
   ////// virtual functions defined in base class
   // begin path modifiers
   void push_tmpl_path(const char *new_comp) {
@@ -196,12 +219,6 @@ private:
   bool get_comment(string& comment, bool active_cfg);
   bool marked_display_default(bool active_cfg);
 
-  // observers during commit operation
-  bool marked_committed(const tr1::shared_ptr<Ctemplate>& def, bool is_set);
-
-  // these operate on both current tmpl and work paths
-  bool validate_val_impl(const tr1::shared_ptr<Ctemplate>& def, char *value);
-
   // observers for "edit/tmpl levels" (for "edit"-related operations).
   // note that these should be moved to base class in the future.
   string get_edit_level_path() {
@@ -215,6 +232,10 @@ private:
     return cfg_path_at_root();
   };
 
+  // functions for commit operation
+  bool marked_committed(bool is_delete);
+  bool mark_committed(bool is_delete);
+
   // for testing/debugging
   string cfg_path_to_str();
   string tmpl_path_to_str();
@@ -226,14 +247,33 @@ private:
   void push_path(FsPath& old_path, const char *new_comp);
   void pop_path(FsPath& path);
   void pop_path(FsPath& path, string& last);
-  void get_all_child_dir_names(const FsPath& root, vector<string>& nodes);
-  bool write_file(const FsPath& file, const string& data);
+  bool check_dir_entries(const FsPath& root, vector<string> *cnodes,
+                         bool filter_nodes = true, bool empty_check = false);
+  bool is_directory_empty(const FsPath& d) {
+    return (!check_dir_entries(d, NULL, false, true));
+  }
+  void get_all_child_dir_names(const FsPath& root, vector<string>& nodes) {
+    check_dir_entries(root, &nodes);
+  }
+  bool write_file(const char *file, const string& data,
+                  bool append = false);
+  bool write_file(const FsPath& file, const string& data,
+                  bool append = false) {
+    return write_file(file.path_cstr(), data, append);
+  }
+  bool create_file(const char *file) {
+    return write_file(file, "");
+  };
   bool create_file(const FsPath& file) {
     return write_file(file, "");
   };
   bool read_whole_file(const FsPath& file, string& data);
-  bool committed_marker_exists(const string& marker);
-  void recursive_copy_dir(const FsPath& src, const FsPath& dst);
+  void recursive_copy_dir(const FsPath& src, const FsPath& dst,
+                          bool filter_dot_entries = false);
+  void get_committed_marker(bool is_delete, string& marker);
+  bool find_line_in_file(const FsPath& file, const string& line);
+  bool do_mount(const FsPath& rwdir, const FsPath& rdir, const FsPath& mdir);
+  bool do_umount(const FsPath& mdir);
 
   // boost fs operations wrappers
   bool b_fs_get_file_status(const char *path, b_fs::file_status& fs) {
diff --git a/src/cstore/unionfs/fspath.hpp b/src/cstore/unionfs/fspath.hpp
index 35985ed..fbaafd6 100644
--- a/src/cstore/unionfs/fspath.hpp
+++ b/src/cstore/unionfs/fspath.hpp
@@ -65,6 +65,7 @@ public:
   };
 
   size_t length() const { return _data.length(); };
+  size_t size() const { return _data.size(); };
   bool has_parent_path() const { return (_data.size() > 0); };
   const char *path_cstr() const { return _data.get_cstr(); };
   size_t hash() const { return _data.hash(); };