/*
 *
  ***** BEGIN LICENSE BLOCK *****
 
  Copyright (C) 2009-2016 Olof Hagsand and Benny Holmgren
  Copyright (C) 2017-2019 Olof Hagsand
  Copyright (C) 2020-2022 Olof Hagsand and Rubicon Communications, LLC(Netgate)

  This file is part of CLIXON.

  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.

  Alternatively, the contents of this file may be used under the terms of
  the GNU General Public License Version 3 or later (the "GPL"),
  in which case the provisions of the GPL are applicable instead
  of those above. If you wish to allow use of your version of this file only
  under the terms of the GPL, and not to allow others to
  use your version of this file under the terms of Apache License version 2, 
  indicate your decision by deleting the provisions above and replace them with
  the  notice and other provisions required by the GPL. If you do not delete
  the provisions above, a recipient may use your version of this file under
  the terms of any one of the Apache License version 2 or the GPL.

  ***** END LICENSE BLOCK *****

 *
 * Translation / mapping code between formats
 */
#ifdef HAVE_CONFIG_H
#include "clixon_config.h" /* generated by config & autoconf */
#endif

#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <errno.h>
#include <ctype.h>
#include <string.h>
#include <syslog.h>
#include <fcntl.h>
#include <assert.h>
#include <arpa/inet.h>
#include <sys/param.h>
#include <netinet/in.h>

/* cligen */
#include <cligen/cligen.h>

/* clicon */

#include "clixon_string.h"
#include "clixon_queue.h"
#include "clixon_hash.h"
#include "clixon_handle.h"
#include "clixon_string.h"
#include "clixon_yang.h"
#include "clixon_xml.h"
#include "clixon_options.h"
#include "clixon_data.h"
#include "clixon_yang_module.h"
#include "clixon_plugin.h"
#include "clixon_xml_nsctx.h"
#include "clixon_xpath_ctx.h"
#include "clixon_xpath.h"
#include "clixon_log.h"
#include "clixon_err.h"
#include "clixon_netconf_lib.h"
#include "clixon_xml_sort.h"
#include "clixon_yang_type.h"
#include "clixon_text_syntax.h"
#include "clixon_xml_io.h"
#include "clixon_xml_map.h"

/* Local types 
 */
/* Merge code needs a two-phase pass where objects subject to merge are first checked for,
 * the actually inserted.
 * This is to mitigate a search problem where objects inserted are among the ones checked for
 */
typedef struct  {
    cxobj     *mt_x0c;
    cxobj     *mt_x1c;
    yang_stmt *mt_yc;
} merge_twophase;

/*! Is attribute and is either of form xmlns="", or xmlns:x="" */
int
isxmlns(cxobj *x)
{
    char *prefix = NULL;

    if (xml_type(x) != CX_ATTR)
        return 0;
    if (strcmp(xml_name(x), "xmlns")==0)
        return 1;
    if ((prefix = xml_prefix(x)) != NULL
        && strcmp(xml_prefix(x), "xmlns")==0)
        return 1;
    return 0;
}

/*! Translate a single xml node to a cligen variable vector. Note not recursive 
 * @param[in]  xt   XML tree containing one top node
 * @param[in]  ys   Yang spec containing type specification of top-node of xt
 * @param[out] cvv  CLIgen variable vector. Should be freed by cvec_free()
 * @retval     0    Everything OK, cvv allocated and set
 * @retval    -1    Something wrong, clicon_err() called to set error. No cvv returned
 * @note  cvv Should be freed by cvec_free() after use.
 * 'Not recursive' means that only one level of XML bodies is translated to cvec:s.
 * If range is wriong (eg 1000 for uint8) a warning is logged, the value is 
 * skipped, and continues.
 * yang is needed to know which type an xml element has.
 * Example: 
    <a>
      <b>23</b>
      <c>88</c>
      <d>      
        <e>99</e>
      </d>
    </a> 
         --> b:23, c:88
 * @see cvec2xml
 */
int
xml2cvec(cxobj      *xt, 
         yang_stmt  *yt, 
         cvec      **cvv0)
{
    int               retval = -1;
    cvec             *cvv = NULL;
    cxobj            *xc;         /* xml iteration variable */
    yang_stmt        *ys;         /* yang spec */
    cg_var           *cv;
    cg_var           *ycv;
    char             *body;
    char             *reason = NULL;
    int               ret;
    char             *name;

    xc = NULL;
    /* Tried to allocate whole cvv here, but some cg_vars may be invalid */
    if ((cvv = cvec_new(0)) == NULL){
        clicon_err(OE_UNIX, errno, "cvec_new");
        goto err;
    }
    xc = NULL;
    /* Go through all children of the xml tree */
    while ((xc = xml_child_each(xt, xc, CX_ELMNT)) != NULL){
        name = xml_name(xc);
        if ((ys = yang_find_datanode(yt, name)) == NULL){
            clicon_debug(0, "%s: yang sanity problem: %s in xml but not present in yang under %s",
                         __FUNCTION__, name, yang_argument_get(yt));
            if ((body = xml_body(xc)) != NULL){
                if ((cv = cv_new(CGV_STRING)) == NULL){
                    clicon_err(OE_PLUGIN, errno, "cv_new");
                    goto err;
                }
                cv_name_set(cv, name);
                if ((ret = cv_parse1(body, cv, &reason)) < 0){
                    clicon_err(OE_PLUGIN, errno, "cv_parse %s",name);
                    goto err;
                }
                /* If value is out-of-range, log and skip value, and continue */
                if (ret == 0){
                    clicon_log(LOG_WARNING, "cv_parse %s: %s", name, reason); 
                    if (reason)
                        free(reason);
                }
                else
                    cvec_append_var(cvv, cv); /* Add to variable vector */
                cv_free(cv);
            }
        }
        else if ((ycv = yang_cv_get(ys)) != NULL){
            if ((body = xml_body(xc)) != NULL){
                if ((cv = cv_new(CGV_STRING)) == NULL){
                    clicon_err(OE_PLUGIN, errno, "cv_new");
                    goto err;
                }
                if (cv_cp(cv, ycv) < 0){
                    clicon_err(OE_PLUGIN, errno, "cv_cp");
                    goto err;
                }
                if ((ret = cv_parse1(body, cv, &reason)) < 0){
                    clicon_err(OE_PLUGIN, errno, "cv_parse: %s", name);
                    goto err;
                }
                if (ret == 0){
                    clicon_log(LOG_WARNING, "cv_parse %s: %s", name, reason);
                    if (reason)
                        free(reason);
                }
                else
                    cvec_append_var(cvv, cv); /* Add to variable vector */
                cv_free(cv);
            }
        }
    }
    if (clicon_debug_get() > 1){
        clicon_debug(CLIXON_DBG_DETAIL, "%s cvv:\n", __FUNCTION__);
        cvec_print(stderr, cvv);
    }
    *cvv0 = cvv;
    return 0;
 err:
    if (cvv)
        cvec_free(cvv);
    return retval;
}

/*! Translate a cligen variable vector to an XML tree with depth one 
 * @param[in]   cvv  CLIgen variable vector. Should be freed by cvec_free()
 * @param[in]   toptag    The XML tree in xt will have this XML tag
 * @param[in]   xt   Parent, or NULL
 * @param[out]  xt   Pointer to XML tree containing one top node. Should be freed with xml_free
 * @retval      0    Everything OK, cvv allocated and set
 * @retval     -1    Something wrong, clicon_err() called to set error. No xt returned
 * @see xml2cvec
 * @see cvec2xml   This does more but has an internal xml2cvec translation
*/
int
cvec2xml_1(cvec   *cvv, 
           char   *toptag, 
           cxobj  *xp,
           cxobj **xt0)
{
    int               retval = -1;
    cxobj            *xt = NULL;
    cxobj            *xn;
    cxobj            *xb;
    cg_var           *cv;
    char             *val;
    int               len=0;
    int               i;

    cv = NULL;
    while ((cv = cvec_each(cvv, cv)) != NULL) 
        len++;
    if ((xt = xml_new(toptag, xp, CX_ELMNT)) == NULL)
        goto err;
    if (xml_childvec_set(xt, len) < 0)
        goto err;
    cv = NULL;
    i = 0;
    while ((cv = cvec_each(cvv, cv)) != NULL) {
        if (cv_type_get(cv)==CGV_ERR || cv_name_get(cv) == NULL)
            continue;
        if ((xn = xml_new(cv_name_get(cv), NULL, CX_ELMNT)) == NULL) /* this leaks */
            goto err;
        xml_parent_set(xn, xt);
        xml_child_i_set(xt, i++, xn);
        if ((xb = xml_new("body", xn, CX_BODY)) == NULL) /* this leaks */
            goto err;
        val = cv2str_dup(cv);
        xml_value_set(xb, val); /* this leaks */
        if (val)
            free(val);
    }
    *xt0 = xt;
    return 0;
 err:
    if (xt)
        xml_free(xt);
    return retval;
}

/*! Recursive help function to compute differences between two xml trees
 * @param[in]  x0         First XML tree
 * @param[in]  x1         Second XML tree
 * @param[out] x0vec      Pointervector to XML nodes existing in only first tree
 * @param[out] x0veclen   Length of first vector
 * @param[out] x1vec      Pointervector to XML nodes existing in only second tree
 * @param[out] x1veclen   Length of x1vec vector
 * @param[out] changed_x0 Pointervector to XML nodes changed orig value
 * @param[out] changed_x1 Pointervector to XML nodes changed wanted value
 * @param[out] changedlen Length of changed vector
 * Algorithm to compare two sorted lists A, B:
 *   A 0 1 2 3 5 6
 *   B 0 2 4 5 6
 * Let (a, b) be first elements of (A, B) respectively(*)
 *   a = b :  EITHER leafs: a!=b : add a in changed_x0, b in changed_x1,
 *            OR: Set (A,B) to children of (a,b) and call algorithm recursively
 *         , get next (a,b)
 *   a < b : add a in x0, get next a 
 *   a > b : add b in x1, get next b
 * (*) "comparing" a&b here is made by xml_cmp() which judges equality from a structural
 *     perspective, ie both have the same yang spec, if they are lists, they have the
 *     the same keys. NOT that the values are equal!
 * @see xml_diff  API function, this one is internal and recursive
 */
static int
xml_diff1(cxobj     *x0, 
          cxobj     *x1,
          cxobj   ***x0vec,
          int       *x0veclen,
          cxobj   ***x1vec,
          int       *x1veclen,
          cxobj   ***changed_x0,
          cxobj   ***changed_x1,
          int       *changedlen)
{
    int        retval = -1;
    cxobj     *x0c = NULL; /* x0 child */
    cxobj     *x1c = NULL; /* x1 child */
    yang_stmt *yc0;
    yang_stmt *yc1;
    char      *b0;
    char      *b1;
    int        eq;

    /* Traverse x0 and x1 in lock-step */
    x0c = x1c = NULL;    
    x0c = xml_child_each(x0, x0c, CX_ELMNT);
    x1c = xml_child_each(x1, x1c, CX_ELMNT);
    for (;;){
        if (x0c == NULL && x1c == NULL)
            goto ok;
        else if (x0c == NULL){
            if (cxvec_append(x1c, x1vec, x1veclen) < 0) 
                goto done;
            x1c = xml_child_each(x1, x1c, CX_ELMNT);
            continue;
        }
        else if (x1c == NULL){
            if (cxvec_append(x0c, x0vec, x0veclen) < 0) 
                goto done;
            x0c = xml_child_each(x0, x0c, CX_ELMNT);
            continue;
        }
        /* Both x0c and x1c exists, check if they are yang-equal. */
        eq = xml_cmp(x0c, x1c, 0, 0, NULL);
        if (eq < 0){
            if (cxvec_append(x0c, x0vec, x0veclen) < 0) 
                goto done;
            x0c = xml_child_each(x0, x0c, CX_ELMNT);
            continue;
        }
        else if (eq > 0){
            if (cxvec_append(x1c, x1vec, x1veclen) < 0) 
                goto done;
            x1c = xml_child_each(x1, x1c, CX_ELMNT);
            continue;
        }
        else{ /* equal */
            /* xml-spec NULL could happen with anydata children for example,
             * if so, continute compare children but without yang
             */
            yc0 = xml_spec(x0c);
            yc1 = xml_spec(x1c);
            if (yc0 && yc1 && yc0 != yc1){ /* choice */
                if (cxvec_append(x0c, x0vec, x0veclen) < 0) 
                    goto done;
                if (cxvec_append(x1c, x1vec, x1veclen) < 0) 
                    goto done;
            }
            else
                if (yc0 && yang_keyword_get(yc0) == Y_LEAF){
                    /* if x0c and x1c are leafs w bodies, then they may be changed */
                    b0 = xml_body(x0c);
                    b1 = xml_body(x1c);
                    if (b0 == NULL && b1 == NULL)
                        ;
                    else if (b0 == NULL || b1 == NULL
                             || strcmp(b0, b1) != 0 
                             ){
                        if (cxvec_append(x0c, changed_x0, changedlen) < 0) 
                            goto done;
                        (*changedlen)--; /* append two vectors */
                        if (cxvec_append(x1c, changed_x1, changedlen) < 0) 
                            goto done;
                    }
                }
                else if (xml_diff1(x0c, x1c,   
                                   x0vec, x0veclen, 
                                   x1vec, x1veclen, 
                                   changed_x0, changed_x1, changedlen)< 0)
                    goto done;
        }
        x0c = xml_child_each(x0, x0c, CX_ELMNT);
        x1c = xml_child_each(x1, x1c, CX_ELMNT);
    }
 ok:
    retval = 0;
 done:
    return retval;
}

/*! Compute differences between two xml trees
 *
 * @param[in]  x0         First XML tree
 * @param[in]  x1         Second XML tree
 * @param[out] first      Pointervector to XML nodes existing in only first tree
 * @param[out] firstlen   Length of first vector
 * @param[out] second     Pointervector to XML nodes existing in only second tree
 * @param[out] secondlen  Length of second vector
 * @param[out] changed_x0 Pointervector to XML nodes changed orig value
 * @param[out] changed_x1 Pointervector to XML nodes changed wanted value
 * @param[out] changedlen Length of changed vector
 * @retval     0          OK
 * @retval    -1          Error
 * All xml vectors should be freed after use.
 * @see xml_tree_equal  same algorithm but do not bother with what has changed
 */
int
xml_diff(cxobj     *x0, 
         cxobj     *x1,
         cxobj   ***first,
         int       *firstlen,
         cxobj   ***second,
         int       *secondlen,
         cxobj   ***changed_x0,
         cxobj   ***changed_x1,
         int       *changedlen)
{
    int retval = -1;

    *firstlen = 0;
    *secondlen = 0;    
    *changedlen = 0;
    if (x0 == NULL && x1 == NULL)
        return 0;
    if (x1 == NULL){
        if (cxvec_append(x0, first, firstlen) < 0) 
            goto done;
        goto ok;
    }
    if (x0 == NULL){
        if (cxvec_append(x0, second, secondlen) < 0) 
            goto done;
        goto ok;
    }
    if (xml_diff1(x0, x1,
                  first, firstlen, 
                  second, secondlen, 
                  changed_x0, changed_x1, changedlen) < 0)
        goto done;
 ok:
    retval = 0;
 done:
    return retval;
}

/*! Compute if two XML trees are equal or not
 *
 * @param[in]  x0   First XML tree
 * @param[in]  x1   Second XML tree
 * @retval     1    Not equal
 * @retval     0    Equal
 * @see xml_diff which returns diff sets
 */
int
xml_tree_equal(cxobj     *x0, 
               cxobj     *x1)
{
    int        retval = 1; /* Not equal */
    int        eq;
    yang_stmt *yc0;
    yang_stmt *yc1;
    char      *b0;
    char      *b1;
    cxobj     *x0c = NULL; /* x0 child */
    cxobj     *x1c = NULL; /* x1 child */
    
    /* Traverse x0 and x1 in lock-step */
    x0c = x1c = NULL;    
    x0c = xml_child_each(x0, x0c, CX_ELMNT);
    x1c = xml_child_each(x1, x1c, CX_ELMNT);
    for (;;){
        if (x0c == NULL && x1c == NULL)
            goto ok;
        else if (x0c == NULL){
            goto done;
        }
        else if (x1c == NULL){
            goto done;
        }
        /* Both x0c and x1c exists, check if they are yang-equal. */
        eq = xml_cmp(x0c, x1c, 0, 0, NULL);
        if (eq < 0){
            goto done;
        }
        else if (eq > 0){
            goto done;
        }
        else{ /* equal */
            /* xml-spec NULL could happen with anydata children for example,
             * if so, continute compare children but without yang
             */
            yc0 = xml_spec(x0c);
            yc1 = xml_spec(x1c);
            if (yc0 && yc1 && yc0 != yc1){ /* choice */
                goto done;
            }
            else
                if (yc0 && yang_keyword_get(yc0) == Y_LEAF){
                    /* if x0c and x1c are leafs w bodies, then they may be changed */
                    b0 = xml_body(x0c);
                    b1 = xml_body(x1c);
                    if (b0 == NULL && b1 == NULL)
                        ;
                    else if (b0 == NULL || b1 == NULL
                             || strcmp(b0, b1) != 0 
                             ){
                        goto done;
                    }
                }
                else {
                    eq = xml_tree_equal(x0c, x1c);
                    if (eq)
                        goto done;
                }
        }
        x0c = xml_child_each(x0, x0c, CX_ELMNT);
        x1c = xml_child_each(x1, x1c, CX_ELMNT);
    }
 ok:
    retval = 0;
 done:
    return retval;
}

/*! Prune everything that does not pass test or have at least a child* does not
 *
 * @param[in]   xt      XML tree with some node marked
 * @param[in]   flag    Which flag to test for
 * @param[in]   test    1: test that flag is set, 0: test that flag is not set
 * @param[out]  upmark  Set if a child (recursively) has marked set.
 * The function removes all branches that does not pass the test
 * Purge all nodes that dont have MARK flag set recursively.
 * Save all nodes that is MARK:ed or have at least one (grand*)child that is MARKed
 * @code
 *    xml_tree_prune_flagged_sub(xt, XML_FLAG_MARK, 1, NULL);
 * @endcode
 * @note This function seems a little too complex semantics
 * @see xml_tree_prune_flagged for a simpler variant
 */
int
xml_tree_prune_flagged_sub(cxobj *xt, 
                           int    flag,
                           int    test,
                           int   *upmark)
{
    int        retval = -1;
    int        submark;
    int        mark;
    cxobj     *x;
    cxobj     *xprev;
    int        iskey;
    int        anykey=0;
    yang_stmt *yt;

    mark = 0;
    yt = xml_spec(xt); /* can be null */
    x = NULL;
    xprev = x = NULL;
    while ((x = xml_child_each(xt, x, CX_ELMNT)) != NULL) {
        if (xml_flag(x, flag) == (test?flag:0)){
            /* Pass test */
            mark++;
            xprev = x;
            continue; /* mark and stop here */
        }
        /* If it is key dont remove it yet (see second round) */
        if (yt){
            if ((iskey = yang_key_match(yt, xml_name(x), NULL)) < 0)
                goto done;
            if (iskey){
                anykey++;
                xprev = x; /* skip if this is key */
                continue;
            }
        }
        if (xml_tree_prune_flagged_sub(x, flag, test, &submark) < 0)
            goto done;
        /* if xt is list and submark anywhere, then key subs are also marked
         */
        if (submark)
            mark++;
        else{ /* Safe with xml_child_each if last */
            if (xml_purge(x) < 0)
                goto done;
            x = xprev;
        }
        xprev = x;
    }
    /* Second round: if any keys were found, and no marks detected, purge now */
    if (anykey && !mark){
        x = NULL;
        xprev = x = NULL;
        while ((x = xml_child_each(xt, x, CX_ELMNT)) != NULL) {
            /* If it is key remove it here */
            if (yt){
                if ((iskey = yang_key_match(yt, xml_name(x), NULL)) < 0)
                    goto done;
                if (iskey && xml_purge(x) < 0)
                    goto done;
                x = xprev;
            }
            xprev = x; 
        }
    }
    retval = 0;
 done:
    if (upmark)
        *upmark = mark;
    return retval;
}

/*! Prune everything that passes test
 * @param[in]   xt      XML tree with some node marked
 * @param[in]   flag    Which flag to test for
 * @param[in]   test    1: test that flag is set, 0: test that flag is not set
 * The function removes all branches that does not pass test
 * @code
 *    xml_tree_prune_flagged(xt, XML_FLAG_MARK, 1);
 * @endcode
 */
int
xml_tree_prune_flagged(cxobj *xt, 
                       int    flag,
                       int    test)
{
    int        retval = -1;
    cxobj     *x;
    cxobj     *xprev;

    x = NULL;
    xprev = NULL;
    while ((x = xml_child_each(xt, x, CX_ELMNT)) != NULL) {
        if (xml_flag(x, flag) == (test?flag:0)){        /* Pass test means purge */
            if (xml_purge(x) < 0)
                goto done;
            x = xprev;
            continue; 
        }
        if (xml_tree_prune_flagged(x, flag, test) < 0)
            goto done;
        xprev = x;
    }
    retval = 0;
 done:
    return retval;
}

/*! Prune everything that passes test
 * @param[in]   xt      XML tree with some node marked
 * @param[in]   flags   Flags set
 * @param[in]   mask    Which flags to test for
 * The function removes all branches that does pass test
 * @code
 *    xml_tree_prune_flags(xt, XML_FLAG_MARK, XML_FLAG_MARK|XML_FLAG_DEFAULT);
 * @endcode
 */
int
xml_tree_prune_flags(cxobj *xt,
                       int    flags,
                       int    mask)
{
    int        retval = -1;
    cxobj     *x;
    cxobj     *xprev;

    x = NULL;
    xprev = NULL;
    while ((x = xml_child_each(xt, x, CX_ELMNT)) != NULL) {
        if (xml_flag(x, mask) == flags){        /* Pass test means purge */
            if (xml_purge(x) < 0)
                goto done;
            x = xprev;
            continue;
        }
        if (xml_tree_prune_flags(x, flags, mask) < 0)
            goto done;
        xprev = x;
    }
    retval = 0;
 done:
    return retval;
}


/*! Change namespace of XML node 
 *
 * @param[in]  x         XML node
 * @param[in]  ns        Change to this namespace (if ns does not exist in tree)
 * @param[in]  prefix    If change, use this prefix
 * @param      0         OK
 * @param     -1         Error
 */
int
xml_namespace_change(cxobj *x, 
                    char   *ns,
                    char   *prefix)
{
    int    retval = -1;
    char  *ns0 = NULL;     /* existing namespace */
    char  *prefix0 = NULL; /* existing prefix */
    cxobj *xp;

    ns0 = NULL;
    if (xml2ns(x, xml_prefix(x), &ns0) < 0)
       goto done;
    if (ns0 && strcmp(ns0, ns) == 0)
        goto ok; /* Already has right namespace */ 
    /* Is namespace already declared? */
    if (xml2prefix(x, ns, &prefix0) == 1){
       /* Yes it is declared and the prefix is prefix0 */
       if (xml_prefix_set(x, prefix0) < 0)
           goto done;
    }
    else{ /* Namespace does not exist, add it */
        /* Clear old prefix if any */
       if (xml_prefix_set(x, NULL) < 0)
           goto done;
       if (xml_type(x) == CX_ELMNT) /* If not element, do the namespace addition to the element */
           xp = x;
       else
           xp = xml_parent(x);
       if (xml_add_namespace(x, xp, prefix, ns) < 0)
           goto done;      
       /* Add prefix to x, if any */
       if (prefix && xml_prefix_set(x, prefix) < 0)
           goto done;
    }
 ok:
    retval = 0;
 done:
    return retval;
}

/*! Sanitize an xml tree: xml node has matching yang_stmt pointer 
 * @param[in]   xt      XML top of tree
 */
int
xml_sanity(cxobj *xt, 
           void  *arg)
{
    int        retval = -1;
    yang_stmt *ys;
    char      *name;

    if ((ys = (yang_stmt*)xml_spec(xt)) == NULL){
        retval = 0;
        goto done;
    }
    name = xml_name(xt);
    if (strstr(yang_argument_get(ys), name)==NULL){
        clicon_err(OE_XML, 0, "xml node name '%s' does not match yang spec arg '%s'", 
                   name, yang_argument_get(ys));
        goto done;
    }
    retval = 0;
 done:
    return retval;
}

/*! Detect state data: Either mark or break on first occurence and return xerror
 * @param[in]   xt      XML tree 
 * @param[out]  xerr    If set return netconf error, abort and return if a state variable found
 * @retval      -1      Error
 * @retval      0       Status node found and return xerror
 * @retval      1       OK
 * Note that the behaviour is quite different if xerr is set or not,...
 */
int
xml_non_config_data(cxobj  *xt, 
                    cxobj **xerr) 
{
    int        retval = -1;
    cxobj     *x;
    yang_stmt *y;
    int        ret;
    cbuf      *cb = NULL;
    
    x = NULL;
    while ((x = xml_child_each(xt, x, CX_ELMNT)) != NULL) {
        if ((y = (yang_stmt*)xml_spec(x)) == NULL)
            goto ok;
        if (!yang_config(y)){ /* config == false means state data */
            if (xerr){        /* behaviour 1: return on error */
                if ((cb = cbuf_new()) == NULL){
                    clicon_err(OE_UNIX, errno, "cbuf_new");
                    goto done;
                }
                cprintf(cb, "module %s: state data node unexpected", yang_argument_get(ys_module(y)));
                if (netconf_bad_element_xml(xerr, "application", yang_argument_get(y), cbuf_get(cb)) < 0)
                    goto done;
                retval = 0;
                goto done;
            }
            xml_flag_set(x, XML_FLAG_MARK); /* behaviour 2: mark and continue */
        }
        if ((ret = xml_non_config_data(x, xerr)) < 0)
            goto done;
        if (ret == 0){
            retval = 0;
            goto done;
        }
    }
 ok:
    retval = 1;
 done:
    if (cb)
        cbuf_free(cb);
    return retval;
}

/*! Check if the module tree x is in is assigned right XML namespace, assign if not
 * @param[in]  x   XML node
 *(0. You should probably find the XML root and apply this function to that.)
 * 1. Check which namespace x should have (via yang). This is correct namespace.
 * 2. Check which namespace x has via its XML tree
 * 3. If equal, OK
 * 4. Assign the correct namespace to the XML node
 * Assign default namespace to x. Create an "xmlns"
 * @code
 *    xml_yang_root(x, &xroot);
 *    xmlns_assign(xroot);
 * @endcode
 */
int
xmlns_assign(cxobj *x)
{
    int        retval = -1;
    yang_stmt *y;
    char      *ns_correct; /* correct uri */
    char      *ns_xml;     /* may be null or incorrect */

    if ((y = xml_spec(x)) == NULL){
        clicon_err(OE_YANG, ENOENT, "XML %s does not have yang spec", xml_name(x));
        goto done;
    }
    /* 1. Check which namespace x should have (via yang). This is correct namespace. */
    if ((ns_correct = yang_find_mynamespace(y)) == NULL){
        clicon_err(OE_YANG, ENOENT, "yang node %s does not have namespace", yang_argument_get(y));
        goto done;
    }
    /* 2. Check which namespace x has via its XML tree */
    if (xml2ns(x, NULL, &ns_xml) < 0)
        goto done;
    /* 3. If equal, OK, 4. Else, find root of XML tree */
    if (ns_xml && strcmp(ns_xml, ns_correct)==0)
        goto ok;
    /* 4. Assign the correct namespace */
    if (xmlns_set(x, NULL, ns_correct) < 0)
        goto done;
 ok:
    retval = 0;
 done:
    return retval;
}

/*! Given a src element node x0 and a target node x1, assign (optional) prefix and namespace
 * @param[in]  x1       XML tree
 * @param[in]  x1p      XML tree parent
 * @retval     0        OK
 * @retval     -1       OK
 * @see assign_namespace_element  this is a subroutine
 */
static int
assign_namespace(cxobj     *x1, /* target */
                 cxobj     *x1p,
                 int        isroot,
                 char      *ns,
                 char      *prefix0)
{
    int    retval = -1;
    char  *prefix1 = NULL;
    char  *pexist = NULL;
    cvec  *nsc0 = NULL;
    cvec  *nsc = NULL;
    
    /* 2a. Detect if namespace is declared in x1 target parent */
    if (xml2prefix(x1p, ns, &pexist) == 1){
        /* Yes, and it has prefix pexist */
        if (pexist){
            if ((prefix1 = strdup(pexist)) == NULL){
                clicon_err(OE_UNIX, errno, "strdup");
                goto done;
            }
        }
        else
            prefix1 = NULL;
        /* 3. If yes, assign prefix to x1 */
        if (prefix1 && xml_prefix_set(x1, prefix1) < 0)
            goto done;
        /* And copy namespace context from parent to child */
        if ((nsc0 = nscache_get_all(x1p)) != NULL){
            if ((nsc = cvec_dup(nsc0)) == NULL){
                clicon_err(OE_UNIX, errno, "cvec_dup");
                goto done;
            }
            nscache_replace(x1, nsc);
        }
        /* Just in case */
        if (nscache_set(x1, prefix1, ns) < 0)
            goto done;
    }
    else{ /* No, namespace does not exist in x1 _parent_ 
           * Check if it is exists in x1 itself */
        if (xml2prefix(x1, ns, &pexist) == 1){
            /* Yes it exists, but is it equal? */
            if (clicon_strcmp(pexist, prefix0) == 0)
                ; /* Equal, reuse */
            else{ /* namespace exist, but not equal, use existing */
                /* Add prefix to x1, if any */
                if (pexist && xml_prefix_set(x1, pexist) < 0)
                    goto done;
            }
            goto ok; /* skip */
        }
        else { /* namespace does not exist in target x1, 
                */
            if (isroot){
                if (prefix0 && (prefix1 = strdup(prefix0)) == NULL){
                    clicon_err(OE_UNIX, errno, "strdup");
                    goto done;
                }
            }
            else{
                if (prefix0 == NULL){ /* Use default namespace, may break use of previous default 
                                       * somewhere in x1
                                       */
                    prefix1 = NULL;
                }
            }
        }
        if (xml_add_namespace(x1, x1, prefix1, ns) < 0)
            goto done;
        if (prefix1 && xml_prefix_set(x1, prefix1) < 0)
            goto done;      
    }
 ok:
    /* 6. Ensure x1 cache is updated (I think it is done w xmlns_set above) */
    retval = 0;
 done:
    if (prefix1)
        free(prefix1);
    return retval;
}

/*! Given a src element node x0 and a target node x1, assign (optional) prefix and namespace
 * @param[in]  x0       Source XML tree
 * @param[in]  x1       Target XML tree
 * @param[in]  x1p      Target XML tree parent
 * @retval     0        OK
 * @retval     -1       OK
 * 1. Find N=namespace(x0)
 * 2. Detect if N is declared in x1 parent
 * 3. If yes, assign prefix to x1
 * 4. If no, if default namespace use that, otherwise create new prefix/namespace binding and assign 
 *    that to x1p
 * 5. Add prefix to x1, if any
 * 6. Ensure x1 cache is updated
 * @note switch use of x0 and x1 compared to datastore text_modify
 * @see xml2ns
*/
int
assign_namespace_element(cxobj *x0, /* source */
                         cxobj *x1, /* target */
                         cxobj *x1p)
{
    int        retval = -1;
    char      *namespace = NULL;
    char      *prefix0 = NULL;;
    int        isroot;
    
    isroot = xml_parent(x1p)==NULL &&
        xml_flag(x1p, XML_FLAG_TOP) &&
        xml_prefix(x1p)==NULL;
    /* 1. Find N=namespace(x0) in x0 element */
    prefix0 = xml_prefix(x0);
    if (xml2ns(x0, prefix0, &namespace) < 0)
        goto done;
    if (namespace == NULL){
        clicon_err(OE_XML, ENOENT, "No namespace found for prefix:%s",
                   prefix0?prefix0:"NULL");
        goto done;
    }
    if (assign_namespace(x1, x1p, isroot, namespace, prefix0) < 0)
        goto done;
    /* 6. Ensure x1 cache is updated (I think it is done w xmlns_set above) */
    retval = 0;
 done:
    return retval;
}

/*! Copy namespace declarations from source to target
 *
 * If origin body has namespace definitions, copy them. The reason is that
 * some bodies rely on namespace prefixes, such as NACM path, but there is 
 * no way we can know this here.
 * However, this may lead to namespace collisions if these prefixes are not
 * canonical, and may collide with the assign_namespace_element() above (but that 
 * is for element symbols)
 *
 * @param[in]  x0     Source XML
 * @param[in]  x1     Destination XML
 * @note "standard" namespaces, including clixon internal namespaces are removed
 */
int
assign_namespace_body(cxobj *x0, /* source */
                      cxobj *x1) /* target */
{
    int    retval = -1;
    char  *namespace = NULL;
    char  *namespace1;
    char  *name;
    char  *prefix0;         
    char  *prefix1 = NULL;  
    cxobj *xa;
    
    xa = NULL;
    while ((xa = xml_child_each_attr(x0, xa)) != NULL) {
        prefix0 = xml_prefix(xa);
        name = xml_name(xa);
        namespace = xml_value(xa);
        if ((strcmp(name, "xmlns")==0 && prefix0==NULL) ||
            (prefix0 != NULL && strcmp(prefix0, "xmlns")==0)){
            if (prefix0 == NULL)
                prefix1 = NULL;
            else
                prefix1 = name;
            /* prefix1 contains actual prefix or NULL, prefix0 can be xmlns */
            if (strcmp(namespace, NETCONF_BASE_NAMESPACE) == 0 ||
                strcmp(namespace, YANG_XML_NAMESPACE) == 0 ||
                strcmp(namespace, CLIXON_CONF_NS) == 0 ||
                strcmp(namespace, CLIXON_LIB_NS) == 0
                )
                continue;
            continue; // XXX
            /* Detect if prefix:namespace is declared already? */
            if (xml2ns(x1, prefix1, &namespace1) == 1)
                continue;
            /* Does prefix already point at right namespace? */
            if (namespace1 && strcmp(namespace, namespace1)==0)
                continue;
            /* No, add entry */
            if (xml_add_namespace(x1, x1, prefix1, namespace) < 0)
                goto done;
        }
    }
    /* 6. Ensure x1 cache is updated (I think it is done w xmlns_set above) */
    retval = 0;
 done:
    return retval;
}

/*! Merge a base tree x0 with x1 with yang spec y
 * @param[in]  x0  Base xml tree (can be NULL in add scenarios)
 * @param[in]  y0  Yang spec corresponding to xml-node x0. NULL if x0 is NULL
 * @param[in]  x0p Parent of x0
 * @param[in]  x1  xml tree which modifies base
 * @param[out] reason If retval=0 a malloced string
 * @retval     1      OK
 * @retval     0      Yang error, reason is set
 * @retval    -1      Error
 * Assume x0 and x1 are same on entry and that y is the spec
 */
static int
xml_merge1(cxobj              *x0,  /* the target */
           yang_stmt          *y0,
           cxobj              *x0p,
           cxobj              *x1,  /* the source */
           char              **reason)
{
    int             retval = -1;
    char           *x1cname; /* child name */
    cxobj          *x0c; /* base child */
    cxobj          *x0b; /* base body */
    cxobj          *x1c; /* mod child */
    char           *x1bstr; /* mod body string */
    yang_stmt      *yc;  /* yang child */
    cbuf           *cbr = NULL; /* Reason buffer */
    int             ret;
    int             i;
    merge_twophase *twophase = NULL;
    int             twophase_len;
    cvec           *nsc = NULL;
    cg_var         *cv;
    char           *ns;
    char           *px;
    char           *pxe;
    
    if (x1 == NULL || xml_type(x1) != CX_ELMNT || y0 == NULL){
        clicon_err(OE_XML, EINVAL, "x1 is NULL or not XML element, or lacks yang spec");
        goto done;
    }
    if (x0 == NULL){
        if (xml_nsctx_node(x1, &nsc) < 0)
            goto done;
        if (xml_rm(x1) < 0)
            goto done;
        /* This is to make the anydata case a little more robust, more could be done */
        if (xml_spec(x1) == NULL){
            if (xml_addsub(x0p, x1) < 0)
                goto done;
        }
        else
            if (xml_insert(x0p, x1, INS_LAST, NULL, NULL) < 0)
                goto done;
        cv = NULL;
        while ((cv = cvec_each(nsc, cv)) != NULL){
            px = cv_name_get(cv);
            ns = cv_string_get(cv);
            /* Check if namespace exists */
            if ((ret = xml2prefix(x1, ns, &pxe)) < 0)
                goto done;
            if (ret == 0 ||  /* Not exist */
                clicon_strcmp(px, pxe) != 0){ /* Exists and not equal (can be NULL) */
                if (xmlns_set(x1, px, ns) < 0)
                    goto done;
                xml_sort(x1);
            }
        }
        goto ok;
    }
    if (yang_keyword_get(y0) == Y_LEAF_LIST || yang_keyword_get(y0) == Y_LEAF){
        x1bstr = xml_body(x1);
        if (x1bstr){
            if ((x0b = xml_body_get(x0)) == NULL){
                if ((x0b = xml_new("body", x0, CX_BODY)) == NULL)
                    goto done; 
            }
            if (xml_value_set(x0b, x1bstr) < 0)
                goto done;
        }
        if (xml_parent(x0) == NULL &&
            xml_insert(x0p, x0, INS_LAST, NULL, NULL) < 0) 
            goto done;
        if (assign_namespace_element(x1, x0, x0p) < 0)
            goto done;
    } /* if LEAF|LEAF_LIST */
    else { /* eg Y_CONTAINER, Y_LIST  */
        if (assign_namespace_element(x1, x0, x0p) < 0)
            goto done;
        twophase_len = xml_child_nr(x1);
        if ((twophase = calloc(twophase_len, sizeof(*twophase))) == NULL){
            clicon_err(OE_UNIX, errno, "calloc");
            goto done;
        }
        i = 0;
        /* Loop through children of the modification tree */
        x1c = NULL;
        while ((x1c = xml_child_each(x1, x1c, CX_ELMNT)) != NULL) {
            x1cname = xml_name(x1c);
            /* Get yang spec of the child */
            if ((yc = yang_find_datanode(y0, x1cname)) == NULL){
                if (reason){
                    if ((cbr = cbuf_new()) == NULL){
                        clicon_err(OE_XML, errno, "cbuf_new");
                        goto done;
                    }
                    cprintf(cbr, "XML node %s/%s has no corresponding yang specification (Invalid XML or wrong Yang spec?", xml_name(x1), x1cname);
                    if ((*reason = strdup(cbuf_get(cbr))) == NULL){
                        clicon_err(OE_UNIX, errno, "strdup");
                        goto done;
                    }
                }
                goto fail;
            }
            /* See if there is a corresponding node in the base tree */
            x0c = NULL;
            if (yc && match_base_child(x0, x1c, yc, &x0c) < 0)
                goto done;
            /* If x0 already has a value, do not replace it with a default value in x1 */
            if (x0c && xml_flag(x1c, XML_FLAG_DEFAULT))
                continue;
            /* Save x0c, x1c, yc and merge in second wave, so that x1c entries dont "interfer"
             * with itself, ie that later searches are among earlier objects already added
             * to x0 */
            twophase[i].mt_x0c = x0c;
            twophase[i].mt_x1c = x1c;
            twophase[i].mt_yc  = yc;
            i++;
        } /* while */
        twophase_len = i; /* Inital length included non-elements */
        /* Second run where actual merging is done 
         * Loop through children of the modification tree */
        for (i=0; i<twophase_len; i++){
            assert(twophase[i].mt_x1c);
            if ((ret = xml_merge1(twophase[i].mt_x0c,
                           twophase[i].mt_yc,
                           x0,
                           twophase[i].mt_x1c,
                                  reason)) < 0)
                goto done;
            if (ret == 0)
                goto fail;
        }
        if (xml_parent(x0) == NULL &&
            xml_insert(x0p, x0, INS_LAST, NULL, NULL) < 0) 
            goto done;
    } /* else Y_CONTAINER  */
 ok:
    retval = 1;
 done:
    if (nsc)
        cvec_free(nsc);
    if (twophase)
        free(twophase);
    if (cbr)
        cbuf_free(cbr);
    return retval;
 fail:
    retval = 0;
    goto done;
}

/*! Merge XML trees x1 into x0 according to yang spec yspec
 * @param[in]  x0     Base xml tree (can be NULL in add scenarios)
 * @param[in]  x1     xml tree which modifies base
 * @param[in]  yspec  Yang spec
 * @param[out] reason If retval=0, reason is set. Malloced. Needs to be freed by caller
 * @retval     1      OK
 * @retval     0      Yang error, reason is set
 * @retval    -1      Error
 * @note both x0 and x1 need to be top-level trees
 */
int
xml_merge(cxobj     *x0,
          cxobj     *x1,
          yang_stmt *yspec,
          char     **reason)
{
    int        retval = -1;
    char      *x1cname; /* child name */
    cxobj     *x0c; /* base child */
    cxobj     *x1c; /* mod child */
    yang_stmt *yc;
    yang_stmt *ymod;
    cbuf      *cbr = NULL; /* Reason buffer */
    int        i;
    merge_twophase *twophase = NULL;
    int        twophase_len;
    int        ret;

    if (x0 == NULL || x1 == NULL){
        clicon_err(OE_UNIX, EINVAL, "parameters x0 or x1 is NULL");
        goto done;
    }
    twophase_len = xml_child_nr(x1);
    if ((twophase = calloc(twophase_len, sizeof(*twophase))) == NULL){
        clicon_err(OE_UNIX, errno, "calloc");
        goto done;
    }
    /* Loop through children of the modification tree */
    i = 0;
    x1c = NULL;
    while ((x1c = xml_child_each(x1, x1c, CX_ELMNT)) != NULL) {
        x1cname = xml_name(x1c);
        if ((ys_module_by_xml(yspec, x1c, &ymod)) < 0)
            goto done;
        if (ymod == NULL){
            if (reason &&
                (*reason = strdup("Namespace not found or yang spec not loaded")) == NULL){
                    clicon_err(OE_UNIX, errno, "strdup");
                    goto done;
            }
            goto fail;  
        }
        /* Get yang spec of the child */
        if ((yc = yang_find_datanode(ymod, x1cname)) == NULL){
            if (reason){
                if ((cbr = cbuf_new()) == NULL){
                    clicon_err(OE_XML, errno, "cbuf_new");
                    goto done;
                }
                cprintf(cbr, "XML node %s/%s has no corresponding yang specification (Invalid XML or wrong Yang spec?)", xml_name(x1), x1cname);
                if ((*reason = strdup(cbuf_get(cbr))) == NULL){
                    clicon_err(OE_UNIX, errno, "strdup");
                    goto done;
                }
            }
            goto fail;
        }
        x0c = NULL;
        /* See if there is a corresponding node (x1c) in the base tree (x0) */
        if (yc && match_base_child(x0, x1c, yc, &x0c) < 0)
            goto done;
        /* If x0 already has a value, do not replace it with a default value in x1 */
        if (x0c && xml_flag(x1c, XML_FLAG_DEFAULT))
            continue;
        /* Save x0c, x1c, yc and merge in second wave, so that x1c entries don't "interfere"
         * with itself, ie that later searches are among earlier objects already added
         * to x0 */
        twophase[i].mt_x0c = x0c;
        twophase[i].mt_x1c = x1c;
        twophase[i].mt_yc  = yc;
        i++;
    }
    twophase_len = i; /* Inital length included non-elements */
    /* Second run where actual merging is done 
     * Loop through children of the modification tree */
    for (i=0; i<twophase_len; i++){
        assert(twophase[i].mt_x1c);
        if ((ret = xml_merge1(twophase[i].mt_x0c,
                              twophase[i].mt_yc,
                              x0,
                              twophase[i].mt_x1c,
                              reason)) < 0)
            goto done;
        if (ret == 0)
            goto fail;
    }
    retval = 1; /* OK */
 done:
    if (twophase)
        free(twophase);
    if (cbr)
        cbuf_free(cbr);
    return retval;
 fail:
    retval = 0;
    goto done;
}

/*! Given a YANG (enum) type node and a value, return the string containing corresponding int str
 *
 * @param[in]  ytype   YANG type noden
 * @param[in]  valstr  Integer string value
 * @param[out] enumstr Value of enum, dont free
 */
int
yang_valstr2enum(yang_stmt *ytype,
                 char      *valstr,
                 char     **enumstr)
{
    int        retval = -1;
    yang_stmt *yenum = NULL;
    yang_stmt *yval; 

    if (enumstr == NULL){
        clicon_err(OE_UNIX, EINVAL, "str is NULL");
        goto done;
    }
    while ((yenum = yn_each(ytype, yenum)) != NULL) {
        if ((yval = yang_find(yenum, Y_VALUE, NULL)) == NULL)
            goto done;
        if (strcmp(yang_argument_get(yval), valstr) == 0)
            break;
    }
    if (yenum)
        *enumstr = yang_argument_get(yenum);
    retval = 0;
 done:
    return retval;
}

/*! Given a YANG (enum) type node and a value, return the string containing corresponding int str
 *
 * @param[in]  ytype   YANG type noden
 * @param[in]  enumstr Value of enum
 * @param[out] valstr  Corresponding string containing an int (direct pointer, dont free)
 * @retval     1       OK, result in valstr
 * @retval     0       Invalid, not found
 * @retval     -1      Error
 */
int
yang_enum2valstr(yang_stmt *ytype,
                 char      *enumstr,
                 char     **valstr)
{
    int        retval = -1;
    yang_stmt *yenum; 
    yang_stmt *yval; 

    if (valstr == NULL){
        clicon_err(OE_UNIX, EINVAL, "valstr is NULL");
        goto done;
    }
    if ((yenum = yang_find(ytype, Y_ENUM, enumstr)) == NULL)
        goto fail;
    /* Should assign value if yval not found */
    if ((yval = yang_find(yenum, Y_VALUE, NULL)) == NULL)
        goto done;
    *valstr = yang_argument_get(yval);
    retval = 1;
 done:
    return retval;
 fail:
    retval = 0;
    goto done;
}

/*! Get integer value from xml node from yang enumeration 
 * @param[in]  node XML node in a tree
 * @param[out] val  Integer value returned
 * @retval     0    OK, value parsed
 * @retval    -1    Error, value not obtained or parsed, no reason given
 * @note assume yang spec set
 * @note The function only returns assigned values, but according to RFC:
   If a value is not specified, then one will be automatically assigned.
   If the "enum" substatement is the first one defined, the assigned
   value is zero (0); otherwise, the assigned value is one greater than
   the current highest enum value (i.e., the highest enum value,
   * Thanks: Matthew Smith
 */
int
yang_enum_int_value(cxobj   *node, 
                    int32_t *val)
{
    int retval = -1;
    yang_stmt *yspec;
    yang_stmt *ys;
    yang_stmt *ytype;
    yang_stmt *yrestype;  /* resolved type */
    char      *reason = NULL;
    char      *intstr = NULL;

    if (node == NULL)
        goto done;
    if ((ys = (yang_stmt *) xml_spec(node)) == NULL)
        goto done;
    if ((yspec = ys_spec(ys)) == NULL)
        goto done;
    if ((ytype = yang_find(ys, Y_TYPE, NULL)) == NULL)
        goto done;
    if (yang_type_resolve(ys, ys, ytype, &yrestype, 
                          NULL, NULL, NULL, NULL, NULL) < 0)
        goto done;
    if (yrestype == NULL){
        clicon_err(OE_YANG, 0, "result-type should not be NULL");
        goto done;
    }
    if (yrestype==NULL || strcmp(yang_argument_get(yrestype), "enumeration"))
        goto done;
    if (yang_enum2valstr(yrestype, xml_body(node), &intstr) < 0)
        goto done;
    /* reason is string containing why int could not be parsed */
    if (parse_int32(intstr, val, &reason) < 0)
        goto done;
    retval = 0;
done:
    return retval;
}

/*! Given XML tree x0 with marked nodes, copy marked nodes to new tree x1
 * Two marks are used: XML_FLAG_MARK and XML_FLAG_CHANGE
 *
 * The algorithm works as following:
 * (1) Copy individual nodes marked with XML_FLAG_CHANGE 
 * until nodes marked with XML_FLAG_MARK are reached, where 
 * (2) the complete subtree of that node is copied. 
 * (3) Special case: key nodes in lists are copied if any node in list is marked
 *  @note you may want to check:!yang_config(ys)
 */
int
xml_copy_marked(cxobj *x0, 
                cxobj *x1)
{
    int        retval = -1;
    int        mark;
    cxobj     *x;
    cxobj     *xcopy;
    int        iskey;
    yang_stmt *yt;
    char      *name;
    char      *prefix;

    assert(x0 && x1);
    yt = xml_spec(x0); /* can be null */
    xml_spec_set(x1, yt);
   /* Copy prefix*/
    if ((prefix = xml_prefix(x0)) != NULL)
        if (xml_prefix_set(x1, prefix) < 0)
            goto done;
    
    /* Copy all attributes */
    x = NULL;
    while ((x = xml_child_each_attr(x0, x)) != NULL) {
        name = xml_name(x);
        if ((xcopy = xml_new(name, x1, CX_ATTR)) == NULL)
            goto done;
        if (xml_copy(x, xcopy) < 0) 
            goto done;
    }

    /* Go through children to detect any marked nodes:
     * (3) Special case: key nodes in lists are copied if any 
     * node in list is marked
     */
    mark = 0;
    x = NULL;
    while ((x = xml_child_each(x0, x, CX_ELMNT)) != NULL) {
        if (xml_flag(x, XML_FLAG_MARK|XML_FLAG_CHANGE)){
            mark++;
            break;
        }
    }
    x = NULL;
    while ((x = xml_child_each(x0, x, CX_ELMNT)) != NULL) {
        name = xml_name(x);
        if (xml_flag(x, XML_FLAG_MARK)){
            /* (2) the complete subtree of that node is copied. */
            if ((xcopy = xml_new(name, x1, CX_ELMNT)) == NULL)
                goto done;
            if (xml_copy(x, xcopy) < 0) 
                goto done;
            continue; 
        }
        if (xml_flag(x, XML_FLAG_CHANGE)){
            /*  Copy individual nodes marked with XML_FLAG_CHANGE */
            if ((xcopy = xml_new(name, x1, CX_ELMNT)) == NULL)
                goto done;
            if (xml_copy_marked(x, xcopy) < 0) /*  */
                goto done;
        }
        /* (3) Special case: key nodes in lists are copied if any 
         * node in list is marked */
        if (mark && yt && yang_keyword_get(yt) == Y_LIST){
            /* XXX: I think yang_key_match is suboptimal here */
            if ((iskey = yang_key_match(yt, name, NULL)) < 0)
                goto done;
            if (iskey){
                if ((xcopy = xml_new(name, x1, CX_ELMNT)) == NULL)
                    goto done;
                if (xml_copy(x, xcopy) < 0) 
                    goto done;
            }
        }
    }
    retval = 0;
 done:
    return retval;
}

/*! Check when condition 
 * 
 * @param[in]   xn     XML node, can be NULL, in which case it is added as dummy under xp
 * @param[in]   xp     XML parent
 * @param[in]   yn     Yang node
 * @param[out]  hit    when statement found
 * @param[out]  nrp    1: when stmt evaluates to true
 * @param[out]  xpathp when stmts xpath
 * @retval      0      OK
 * @retval      -1     Error
 * First variants of WHEN: Augmented and uses when using special info in node
 * Second variant of when, actual "when" sub-node RFC 7950 Sec 7.21.5. Can only be one.
 */
int
yang_check_when_xpath(cxobj        *xn,
                      cxobj        *xp,
                      yang_stmt    *yn,
                      int          *hit,
                      int          *nrp,
                      char        **xpathp)
{
    int        retval = 1;
    yang_stmt *yc;
    char      *xpath = NULL;
    cxobj     *x = NULL;
    int        nr = 0;
    cvec      *nsc = NULL;
    int        xmalloc = 0;   /* ugly help variable to clean temporary object */
    int        nscmalloc = 0; /* ugly help variable to remove */

    /* First variant */
    if ((xpath = yang_when_xpath_get(yn)) != NULL){
        x = xp;
        nsc = yang_when_nsc_get(yn);
        *hit = 1;
    }
    /* Second variant */
    else if ((yc = yang_find(yn, Y_WHEN, NULL)) != NULL){
        xpath = yang_argument_get(yc); /* "when" has xpath argument */
        /* Create dummy */
        if (xn == NULL){
            if ((x = xml_new(yang_argument_get(yn), xp, CX_ELMNT)) == NULL)
                goto done;
            xml_spec_set(x, yn);
            xmalloc++;
        }
        else
            x = xn;
        if (xml_nsctx_yang(yn, &nsc) < 0)
            goto done;
        nscmalloc++;
        *hit = 1;
    }
    else
        *hit = 0;
    if (x && xpath){
        if ((nr = xpath_vec_bool(x, nsc, "%s", xpath)) < 0)
            goto done;
    }
    if (nrp)
        *nrp = nr;
    if (xpathp)
        *xpathp = xpath;
    retval = 0;
 done:
    if (xmalloc)
        xml_purge(x);
    if (nsc && nscmalloc)
        xml_nsctx_free(nsc);
    return retval;
}

/*! Check if node is (recursively) mandatory also checking when conditional
 *
 * @param[in]  xn  Optional XML node
 * @param[in]  xp  XML parent
 * @param[in]  ys  YANG node
 * @retval     1   Recursively contains a mandatory node
 * @retval     0   Does not contain a mandatory node
 * @retval    -1   Error
 * @see RFC7950 Sec 3:
 *   o  mandatory node: A mandatory node is one of:
 *      1)  A leaf, choice, anydata, or anyxml node with a "mandatory"
 *         statement with the value "true".
 *      2) # see below
 *      3)  A container node without a "presence" statement and that has at
 *         least one mandatory node as a child.
 */
int
yang_xml_mandatory(cxobj     *xt,
                   yang_stmt *ys)
{
    int           retval = -1;
    yang_stmt    *ym;
    cg_var       *cv;
    enum rfc_6020 keyw;
    cxobj        *xs = NULL;
    int           ret;
    yang_stmt    *yc;
    int           hit;
    int           nr;

    /* Create dummy xs if not exist */
    if ((xs = xml_new(yang_argument_get(ys), xt, CX_ELMNT)) == NULL)
        goto done;
    xml_spec_set(xs, ys);
    if (yang_check_when_xpath(xs, xt, ys, &hit, &nr, NULL) < 0)
        goto done;
    if (hit && !nr){
        retval = 0;
        goto done;
    }
    keyw = yang_keyword_get(ys);
    if (keyw == Y_LEAF || keyw == Y_CHOICE || keyw == Y_ANYDATA || keyw == Y_ANYXML){
        if ((ym = yang_find(ys, Y_MANDATORY, NULL)) != NULL){
            if ((cv = yang_cv_get(ym)) != NULL){ /* shouldnt happen */
                retval = cv_bool_get(cv);
                goto done;
            }
        }
    }
    /* 3) A container node without a "presence" statement and that has at
     *    least one mandatory node as a child. */
    else if (keyw == Y_CONTAINER && 
             yang_find(ys, Y_PRESENCE, NULL) == NULL){
        yc = NULL;
        while ((yc = yn_each(ys, yc)) != NULL) {
            if ((ret = yang_xml_mandatory(xs, yc)) < 0)
                goto done;
            if (ret == 1)
                goto mandatory;
        }
    }
    retval = 0; /* Does not contain mandatory node */
 done:
    if (xs != NULL)
        xml_purge(xs);
    return retval;
 mandatory:
    retval = 1;
    goto done;
}

/*! Is XML node (ie under <rpc>) an action, ie name action and belong to YANG_XML_NAMESPACE?
 * @param[in]   xn   XML node
 * @retval      1    Yes, an action
 * @retval      0    No, not an action
 * @retval     -1    Error
 */
int
xml_rpc_isaction(cxobj *xn)
{
    int   retval = -1;
    char *ns = NULL;

    if (strcmp(xml_name(xn), "action") != 0)
        goto fail;
    if (xml2ns(xn, xml_prefix(xn), &ns) < 0)
        goto done;
    if (strcmp(YANG_XML_NAMESPACE, ns) != 0)
        goto fail;
    retval = 1; // is action
 done:
    return retval;
 fail:
    retval = 0;
    goto done;
}

/*! Find innermost node under <action> carrying the name of the defined action
 *  Find innermost container or list contains an XML element
 *  that carries the name of the defined action. 
 * @param[in]   xn   XML node
 * @param[in]   top  If set, dont look for action node since top-level
 * @param[out]  xap  Pointer to xml action node
 * @retval      0    OK
 * @retval     -1    Error
 * XXX: does not look at list key
 */
int
xml_find_action(cxobj  *xn,
                int     top,
                cxobj **xap)
{
    int        retval = -1;
    cxobj     *xc = NULL;
    yang_stmt *yc;

    while ((xc = xml_child_each(xn, xc, CX_ELMNT)) != NULL) {
        if ((yc = xml_spec(xc)) == NULL)
            continue;
        if (!top && yang_keyword_get(yc) == Y_ACTION){
            *xap = xc;
            break;
        }
        if (yang_keyword_get(yc) != Y_CONTAINER && yang_keyword_get(yc) != Y_LIST)
            continue;
        /* XXX check key */
        if (xml_find_action(xc, 0, xap) < 0)
            goto done;
        break;
    }
    retval = 0;
 done:
    return retval;
}

/*! Utility function: recursive traverse an XML tree and remove nodes based on attribute value
 * Conditionally remove attribute and node
 * @param[in]  xn       XML node
 * @param[in]  ns       Namespace of attribute
 * @param[in]  name     Attribute name
 * @param[in]  value    Attribute value
 * @param[in]  keepnode 0: remove node associated with attribute; 1: keep node but remove attr
 */
int
purge_tagged_nodes(cxobj *xn,
                   char  *ns,
                   char  *name,
                   char  *value,
                   int    keepnode)
{
    int    retval = -1;
    cxobj *x;
    cxobj *xa;
    cxobj *xprev;
    char  *prefix;
    char  *v;
    int    ret;

    x = NULL;
    xprev = NULL;
    while ((x = xml_child_each(xn, x, CX_ELMNT)) != NULL) {
        if ((ret = xml2prefix(x, ns, &prefix)) < 0)
            goto done;
        if (ret == 0)
            continue;
        if ((xa = xml_find_type(x, prefix, "default", CX_ATTR)) != NULL){
            if (!keepnode &&
                (v = xml_value(xa)) != NULL &&
                strcmp(v, value) == 0){
                xml_purge(x);
                x = xprev;
                continue;
            }
            xml_purge(xa); /* remove attribute regardless */
        }
        if (purge_tagged_nodes(x, ns, name, value, keepnode) < 0)
            goto done;
        xprev = x;
    }
    retval = 0;
 done:
    return retval;
}

/*! Compare two dbs using XML. Write to file and run diff. Independent of YANG
 *
 * @param[in]  xc1     XML tree 1
 * @param[in]  xc2     XML tree 2
 * @param[in]  format  "text"|"xml"|"json"|"cli"|"netconf" (see format_enum)
 * @see xml_tree_diff_print with better XML in-mem comparison but is YANG dependent
 */
int
clixon_compare_xmls(cxobj            *xc1,
                    cxobj            *xc2,
                    enum format_enum  format)
{
    int    retval = -1;
    int    fd;
    FILE  *f;
    char   filename1[MAXPATHLEN];
    char   filename2[MAXPATHLEN];
    cbuf  *cb = NULL;

    snprintf(filename1, sizeof(filename1), "/tmp/cliconXXXXXX");
    snprintf(filename2, sizeof(filename2), "/tmp/cliconXXXXXX");
    if ((fd = mkstemp(filename1)) < 0){
        clicon_err(OE_UNDEF, errno, "tmpfile");
        goto done;
    }
    if ((f = fdopen(fd, "w")) == NULL)
        goto done;
    switch(format){
    case FORMAT_TEXT:
        if (clixon_txt2file(f, xc1, 0, cligen_output, 1, 1) < 0)
            goto done;
        break;
    case FORMAT_XML:
    default:
        if (clixon_xml2file(f, xc1, 0, 1, NULL, cligen_output, 1, 1) < 0)
            goto done;
        break;
    }
    fclose(f);
    close(fd);

    if ((fd = mkstemp(filename2)) < 0){
        clicon_err(OE_UNDEF, errno, "mkstemp: %s", strerror(errno));
        goto done;
    }
    if ((f = fdopen(fd, "w")) == NULL)
        goto done;

    switch(format){
    case FORMAT_TEXT:
        if (clixon_txt2file(f, xc2, 0, cligen_output, 1, 1) < 0)
            goto done;
        break;
    case FORMAT_XML:
    default:
        if (clixon_xml2file(f, xc2, 0, 1, NULL, cligen_output, 1, 1) < 0)
            goto done;
        break;
    }
    fclose(f);
    close(fd);

    if ((cb = cbuf_new()) == NULL){
        clicon_err(OE_CFG, errno, "cbuf_new");
        goto done;
    }
    cprintf(cb, "diff -dU 1 %s %s |  grep -v @@ | sed 1,2d",
            filename1, filename2);
    if (system(cbuf_get(cb)) < 0)
        goto done;

    retval = 0;
  done:
    if (cb)
        cbuf_free(cb);
    unlink(filename1);
    unlink(filename2);
    return retval;
}

/*! Print diff of two XML trees in memory. YANG dependent
 *
 * Uses underlying XML diff algorithm with better result than clixon_compare_xmls
 * @param[in]  cb      CLIgen buffer
 * @param[in]  level   How many spaces to insert before each line
 * @param[in]  x0      First XML tree
 * @param[in]  x1      Second XML tree
 * @param[in]  format  "text"|"xml"|"json"|"cli"|"netconf" (NYI: only xml)
 * @retval     0       Ok
 * @retval    -1       Error
 * @see xml_diff which returns diff sets
 * @see clixon_compare_xmls which uses files and is independent of YANG
 * XXX only XML
 */
int
xml_tree_diff_print(cbuf             *cb,
                    int               level,
                    cxobj            *x0, 
                    cxobj            *x1,
                    enum format_enum  format)
{
    int        retval = -1;
    cxobj     *x0c = NULL; /* x0 child */
    cxobj     *x1c = NULL; /* x1 child */
    yang_stmt *yc0;
    yang_stmt *yc1;
    char      *b0;
    char      *b1;
    int        eq;
    int        nr=0;

    /* Traverse x0 and x1 in lock-step */
    x0c = x1c = NULL;    
    x0c = xml_child_each(x0, x0c, CX_ELMNT);
    x1c = xml_child_each(x1, x1c, CX_ELMNT);
    for (;;){
        if (x0c == NULL && x1c == NULL)
            goto ok;
        else if (x0c == NULL){
            if (nr==0)
                cprintf(cb, "%*s%s>\n", level*PRETTYPRINT_INDENT, "<", xml_name(x1));
            nr++;
            if (clixon_xml2cbuf(cb, x1c, level+1, 1, "+", -1, 0) < 0)
                goto done;
            x1c = xml_child_each(x1, x1c, CX_ELMNT);
            continue;
        }
        else if (x1c == NULL){
            if (nr==0)
                cprintf(cb, "%*s%s>\n", level*PRETTYPRINT_INDENT, "<", xml_name(x0));
            nr++;
            if (clixon_xml2cbuf(cb, x0c, level+1, 1, "-", -1, 0) < 0)
                goto done;
            x0c = xml_child_each(x0, x0c, CX_ELMNT);
            continue;
        }
        /* Both x0c and x1c exists, check if they are yang-equal. */
        eq = xml_cmp(x0c, x1c, 0, 0, NULL);
        if (eq < 0){
            if (nr==0)
                cprintf(cb, "%*s%s>\n", level*PRETTYPRINT_INDENT, "<", xml_name(x0));
            nr++;
            if (clixon_xml2cbuf(cb, x0c, level+1, 1, "-", -1, 0) < 0)
                goto done;
            x0c = xml_child_each(x0, x0c, CX_ELMNT);
            continue;
        }
        else if (eq > 0){
            if (nr==0)
                cprintf(cb, "%*s%s>\n", level*PRETTYPRINT_INDENT, "<", xml_name(x1));
            nr++;
            if (clixon_xml2cbuf(cb, x1c, level+1, 1, "+", -1, 0) < 0)
                goto done;
            x1c = xml_child_each(x1, x1c, CX_ELMNT);
            continue;
        }
        else{ /* equal */
            /* xml-spec NULL could happen with anydata children for example,
             * if so, continute compare children but without yang
             */
            yc0 = xml_spec(x0c);
            yc1 = xml_spec(x1c);
            if (yc0 && yc1 && yc0 != yc1){ /* choice */
                if (nr==0)
                    cprintf(cb, "%*s%s>\n", level*PRETTYPRINT_INDENT, "<", xml_name(x0));
                nr++;
                if (clixon_xml2cbuf(cb, x0c, level+1, 1, "-", -1, 0) < 0)
                    goto done;
                if (clixon_xml2cbuf(cb, x1c, level+1, 1, "+", -1, 0) < 0)
                    goto done;
            }
            else
                if (yc0 && yang_keyword_get(yc0) == Y_LEAF){
                    /* if x0c and x1c are leafs w bodies, then they may be changed */
                    b0 = xml_body(x0c);
                    b1 = xml_body(x1c);
                    if (b0 == NULL && b1 == NULL)
                        ;
                    else if (b0 == NULL || b1 == NULL
                             || strcmp(b0, b1) != 0 
                             ){
                        if (nr==0)
                            cprintf(cb, "%*s%s>\n", level*PRETTYPRINT_INDENT, "<", xml_name(x0));
                        nr++;
                        cprintf(cb, "-%*s%s>%s</%s>\n", (level*PRETTYPRINT_INDENT-1), "<",
                           xml_name(x0c), b0, xml_name(x0c));
                        cprintf(cb, "+%*s%s>%s</%s>\n", (level*PRETTYPRINT_INDENT-1), "<",
                           xml_name(x1c), b1, xml_name(x1c));
                    }
                }
                else if (xml_tree_diff_print(cb, level+1, x0c, x1c, format) < 0)
                    goto done;
        }
        x0c = xml_child_each(x0, x0c, CX_ELMNT);
        x1c = xml_child_each(x1, x1c, CX_ELMNT);
    }
 ok:
    if (nr)
        cprintf(cb, "%*s/%s>\n", level*PRETTYPRINT_INDENT, "<", xml_name(x0));
    retval = 0;
 done:
    return retval;
}
