/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.hadoop.conf;

import java.lang.Class;
import java.lang.reflect.Field;
import java.lang.reflect.Modifier;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
import static org.junit.Assert.assertTrue;

import org.apache.hadoop.conf.Configuration;

/**
 * Base class for comparing fields in one or more Configuration classes
 * against a corresponding .xml file.  Usage is intended as follows:
 * <p></p>
 * <ol>
 * <li> Create a subclass to TestConfigurationFieldsBase
 * <li> Define <code>initializeMemberVariables</code> method in the
 *      subclass.  In this class, do the following:
 * <p></p>
 *   <ol>
 *   <li> <b>Required</b> Set the variable <code>xmlFilename</code> to
 *        the appropriate xml definition file
 *   <li> <b>Required</b> Set the variable <code>configurationClasses</code>
 *        to an array of the classes which define the constants used by the
 *        code corresponding to the xml files
 *   <li> <b>Optional</b> Set <code>errorIfMissingConfigProps</code> if the
 *        subclass should throw an error in the method
 *        <code>testCompareXmlAgainstConfigurationClass</code>
 *   <li> <b>Optional</b> Set <code>errorIfMissingXmlProps</code> if the
 *        subclass should throw an error in the method
 *        <code>testCompareConfigurationClassAgainstXml</code>
 *   <li> <b>Optional</b> Instantiate and populate strings into one or
 *        more of the following variables:
 *        <br><code>configurationPropsToSkipCompare</code>
 *        <br><code>configurationPrefixToSkipCompare</code>
 *        <br><code>xmlPropsToSkipCompare</code>
 *        <br><code>xmlPrefixToSkipCompare</code>
 *        <br>
 *        in order to get comparisons clean
 *   </ol>
 * </ol>
 * <p></p>
 * The tests to do class-to-file and file-to-class should automatically
 * run.  This class (and its subclasses) are mostly not intended to be
 * overridden, but to do a very specific form of comparison testing.
 */
@Ignore
public abstract class TestConfigurationFieldsBase {

  /**
   * Member variable for storing xml filename.
   */
  protected String xmlFilename = null;

  /**
   * Member variable for storing all related Configuration classes.
   */
  protected Class[] configurationClasses = null;

  /**
   * Throw error during comparison if missing configuration properties.
   * Intended to be set by subclass.
   */
  protected boolean errorIfMissingConfigProps = false;

  /**
   * Throw error during comparison if missing xml properties.  Intended
   * to be set by subclass.
   */
  protected boolean errorIfMissingXmlProps = false;

  /**
   * Set of properties to skip extracting (and thus comparing later) in 
   * extractMemberVariablesFromConfigurationFields.
   */
  protected Set<String> configurationPropsToSkipCompare = null;

  /**
   * Set of property prefixes to skip extracting (and thus comparing later)
   * in * extractMemberVariablesFromConfigurationFields.
   */
  protected Set<String> configurationPrefixToSkipCompare = null;

  /**
   * Set of properties to skip extracting (and thus comparing later) in 
   * extractPropertiesFromXml.
   */
  protected Set<String> xmlPropsToSkipCompare = null;

  /**
   * Set of property prefixes to skip extracting (and thus comparing later)
   * in extractPropertiesFromXml.
   */
  protected Set<String> xmlPrefixToSkipCompare = null;

  /**
   * Member variable to store Configuration variables for later comparison.
   */
  private Map<String,String> configurationMemberVariables = null;

  /**
   * Member variable to store XML properties for later comparison.
   */
  private Map<String,String> xmlKeyValueMap = null;

  /**
   * Member variable to store Configuration variables that are not in the
   * corresponding XML file.
   */
  private Set<String> configurationFieldsMissingInXmlFile = null;

  /**
   * Member variable to store XML variables that are not in the
   * corresponding Configuration class(es).
   */
  private Set<String> xmlFieldsMissingInConfiguration = null;

  /**
   * Member variable for debugging base class operation
   */
  protected boolean configDebug = false;
  protected boolean xmlDebug = false;

  /**
   * Abstract method to be used by subclasses for initializing base
   * members.
   */
  public abstract void initializeMemberVariables();
 
  /**
   * Utility function to extract &quot;public static final&quot; member
   * variables from a Configuration type class.
   *
   * @param fields The class member variables
   * @return HashMap containing <StringValue,MemberVariableName> entries
   */
  private HashMap<String,String>
      extractMemberVariablesFromConfigurationFields(Field[] fields) {
    // Sanity Check
    if (fields==null)
      return null;

    HashMap<String,String> retVal = new HashMap<String,String>();

    // Setup regexp for valid properties
    String propRegex = "^[A-Za-z][A-Za-z0-9_-]+(\\.[A-Za-z0-9_-]+)+$";
    Pattern p = Pattern.compile(propRegex);

    // Iterate through class member variables
    int totalFields = 0;
    String value;
    for (Field f : fields) {
      if (configDebug) {
        System.out.println("Field: " + f);
      }
      // Filter out anything that isn't "public static final"
      if (!Modifier.isStatic(f.getModifiers()) ||
          !Modifier.isPublic(f.getModifiers()) ||
          !Modifier.isFinal(f.getModifiers())) {
        continue;
      }
      // Filter out anything that isn't a string.  int/float are generally
      // default values
      if (!f.getType().getName().equals("java.lang.String")) {
        continue;
      }
      // Convert found member into String
      try {
        value = (String) f.get(null);
      } catch (IllegalAccessException iaException) {
        continue;
      }
      if (configDebug) {
        System.out.println("  Value: " + value);
      }
      // Special Case: Detect and ignore partial properties (ending in x)
      //               or file properties (ending in .xml)
      if (value.endsWith(".xml") ||
          value.endsWith(".")    ||
          value.endsWith("-"))
        continue;
      // Ignore known configuration props
      if (configurationPropsToSkipCompare != null) {
        if (configurationPropsToSkipCompare.contains(value)) {
          continue;
        }
      }
      // Ignore known configuration prefixes
      boolean skipPrefix = false;
      if (configurationPrefixToSkipCompare != null) {
        for (String cfgPrefix : configurationPrefixToSkipCompare) {
	  if (value.startsWith(cfgPrefix)) {
            skipPrefix = true;
            break;
	  }
	}
      }
      if (skipPrefix) {
        continue;
      }
      // Positive Filter: Look only for property values.  Expect it to look
      //                  something like: blah.blah2(.blah3.blah4...)
      Matcher m = p.matcher(value);
      if (!m.find()) {
        if (configDebug) {
          System.out.println("  Passes Regex: false");
        }
        continue;
      }
      if (configDebug) {
        System.out.println("  Passes Regex: true");
      }

      // Save member variable/value as hash
      if (!retVal.containsKey(value)) {
        retVal.put(value,f.getName());
      } else {
        if (configDebug) {
          System.out.println("ERROR: Already found key for property " + value);
        }
      }
    }

    return retVal;
  }

  /**
   * Pull properties and values from filename.
   *
   * @param filename XML filename
   * @return HashMap containing <Property,Value> entries from XML file
   */
  private HashMap<String,String> extractPropertiesFromXml
      (String filename) {
    if (filename==null) {
      return null;
    }

    // Iterate through XML file for name/value pairs
    Configuration conf = new Configuration(false);
    conf.setAllowNullValueProperties(true);
    conf.addResource(filename);

    HashMap<String,String> retVal = new HashMap<String,String>();
    Iterator<Map.Entry<String,String>> kvItr = conf.iterator();
    while (kvItr.hasNext()) {
      Map.Entry<String,String> entry = kvItr.next();
      String key = entry.getKey();
      // Ignore known xml props
      if (xmlPropsToSkipCompare != null) {
        if (xmlPropsToSkipCompare.contains(key)) {
          if (xmlDebug) {
            System.out.println("  Skipping Full Key: " + key);
          }
          continue;
        }
      }
      // Ignore known xml prefixes
      boolean skipPrefix = false;
      if (xmlPrefixToSkipCompare != null) {
        for (String xmlPrefix : xmlPrefixToSkipCompare) {
	  if (key.startsWith(xmlPrefix)) {
	    skipPrefix = true;
            break;
	  }
	}
      }
      if (skipPrefix) {
        if (xmlDebug) {
          System.out.println("  Skipping Prefix Key: " + key);
        }
        continue;
      }
      if (conf.onlyKeyExists(key)) {
        retVal.put(key,null);
        if (xmlDebug) {
          System.out.println("  XML Key,Null Value: " + key);
        }
      } else {
        String value = conf.get(key);
        if (value!=null) {
          retVal.put(key,entry.getValue());
          if (xmlDebug) {
            System.out.println("  XML Key,Valid Value: " + key);
          }
        }
      }
      kvItr.remove();
    }
    return retVal;
  }

  /**
   * Perform set difference operation on keyMap2 from keyMap1.
   *
   * @param keyMap1 The initial set
   * @param keyMap2 The set to subtract
   * @return Returns set operation keyMap1-keyMap2
   */
  private static Set<String> compareConfigurationToXmlFields(Map<String,String> keyMap1, Map<String,String> keyMap2) {
    Set<String> retVal = new HashSet<String>(keyMap1.keySet());
    retVal.removeAll(keyMap2.keySet());
    return retVal;
  }

  /**
   * Initialize the four variables corresponding the Configuration
   * class and the XML properties file.
   */
  @Before
  public void setupTestConfigurationFields() throws Exception {
    initializeMemberVariables();

    // Error if subclass hasn't set class members
    assertTrue(xmlFilename!=null);
    assertTrue(configurationClasses!=null);

    // Create class member/value map
    configurationMemberVariables = new HashMap<String,String>();
    if (configDebug) {
      System.out.println("Reading configuration classes");
      System.out.println("");
    }
    for (Class c : configurationClasses) {
      Field[] fields = c.getDeclaredFields();
      Map<String,String> memberMap =
          extractMemberVariablesFromConfigurationFields(fields);
      if (memberMap!=null) {
        configurationMemberVariables.putAll(memberMap);
      }
    }
    if (configDebug) {
      System.out.println("");
      System.out.println("=====");
      System.out.println("");
    }

    // Create XML key/value map
    if (xmlDebug) {
      System.out.println("Reading XML property files");
      System.out.println("");
    }
    xmlKeyValueMap = extractPropertiesFromXml(xmlFilename);
    if (xmlDebug) {
      System.out.println("");
      System.out.println("=====");
      System.out.println("");
    }

    // Find class members not in the XML file
    configurationFieldsMissingInXmlFile = compareConfigurationToXmlFields
        (configurationMemberVariables, xmlKeyValueMap);

    // Find XML properties not in the class
    xmlFieldsMissingInConfiguration = compareConfigurationToXmlFields
        (xmlKeyValueMap, configurationMemberVariables);
  }

  /**
   * Compares the properties that are in the Configuration class, but not
   * in the XML properties file.
   */
  @Test
  public void testCompareConfigurationClassAgainstXml() {
    // Error if subclass hasn't set class members
    assertTrue(xmlFilename!=null);
    assertTrue(configurationClasses!=null);

    final int missingXmlSize = configurationFieldsMissingInXmlFile.size();

    for (Class c : configurationClasses) {
      System.out.println(c);
    }
    System.out.println("  (" + configurationMemberVariables.size() + " member variables)");
    System.out.println();
    StringBuffer xmlErrorMsg = new StringBuffer();
    for (Class c : configurationClasses) {
      xmlErrorMsg.append(c);
      xmlErrorMsg.append(" ");
    }
    xmlErrorMsg.append("has ");
    xmlErrorMsg.append(missingXmlSize);
    xmlErrorMsg.append(" variables missing in ");
    xmlErrorMsg.append(xmlFilename);
    System.out.println(xmlErrorMsg.toString());
    System.out.println();
    if (missingXmlSize==0) {
      System.out.println("  (None)");
    } else {
      for (String missingField : configurationFieldsMissingInXmlFile) {
        System.out.println("  " + missingField);
      }
    }
    System.out.println();
    System.out.println("=====");
    System.out.println();
    if (errorIfMissingXmlProps) {
      assertTrue(xmlErrorMsg.toString(), missingXmlSize==0);
    }
  }

  /**
   * Compares the properties that are in the XML properties file, but not
   * in the Configuration class.
   */
  @Test
  public void testCompareXmlAgainstConfigurationClass() {
    // Error if subclass hasn't set class members
    assertTrue(xmlFilename!=null);
    assertTrue(configurationClasses!=null);

    final int missingConfigSize = xmlFieldsMissingInConfiguration.size();

    System.out.println("File " + xmlFilename + " (" + xmlKeyValueMap.size() + " properties)");
    System.out.println();
    StringBuffer configErrorMsg = new StringBuffer();
    configErrorMsg.append(xmlFilename);
    configErrorMsg.append(" has ");
    configErrorMsg.append(missingConfigSize);
    configErrorMsg.append(" properties missing in");
    for (Class c : configurationClasses) {
      configErrorMsg.append("  " + c);
    }
    System.out.println(configErrorMsg.toString());
    System.out.println();
    if (missingConfigSize==0) {
      System.out.println("  (None)");
    } else {
      for (String missingField : xmlFieldsMissingInConfiguration) {
        System.out.println("  " + missingField);
      }
    }
    System.out.println();
    System.out.println("=====");
    System.out.println();
    if ( errorIfMissingConfigProps ) {
      assertTrue(configErrorMsg.toString(), missingConfigSize==0);
    }
  }
}
