# Title: z_a_TestCases_CheckOutput_iTHydroPlus.py
# 	Description: This program is designed to test HydroPlus modifications and ensure that deliberate changes have not introduced bugs. 
#	The program calls HydroPlus.exe to test if it will generate output identical to the expected output for known input of test cases. 
#
# Software Requirements: Python 3.6 64-bit
# Input: No command line argument, simply a file in the same directory as the Python script.
#   File: ListOfTests.txt: This file should be placed in the same directory as the Python script, within content below:
#       Executable line: "Executable" key word followed by "," and then path to latest build of HydroPlus.exe; path relative to local directory.
#       Test header line: "TestName" key word followed by "," and then "TestCaseFolderPath" key word; TestName is fixed and searched for in algorithm below
#       Next lines: 1 line for each test with a value for TestName and TestCaseFolderPath ending in backslash, separated by commas
#       TestName = arbitrary name without white space that becomes name of folder holding testing results. 
#       TestCaseFolderPath = path to the known test case folder, with internal and populated folders for input and expectedoutput
#
#   Example ListOfTests.txt file content:
#       Executable,..\HydroPlus\Release\HydroPlus.exe
#       TestName,TestCaseFolderPathWithEndingBackslash
#       Hydro_Statistical_Exp_TI,TestCases\StatHydro\expIR_defaultParams\
#
# Output: Placed in folder labeled Testing_%Y_%m_%d_%H_%M_%S folder created in same directory where Python script was launched
#   File: TestResults.txt file details of each test outcome (Fail or Pass) and summary of all tests. 
#   Sub-Folders: TestName folders for each test, with input, output, and expectedoutput folders filled with content
#
# How to Run: 
#	Open Windows Command Prompt, go to directory with Python script, e.g.,C:\>cd C:\iTree\HydroPlus\TestingFilesAndScript. 
#	C:\iTree\HydroPlus\TestingFilesAndScript>python z_a_TestCases_CheckOutput_iTHydroPlus.py
# 		
# Created: i-Tree coding employee Shannon Conley, 2018 while working to implement standard operating procedure algorithms of with R. Coville and T. Endreny
# Updated:
# 2019/10/10:   T. Endreny, Updated directions, commented all code, refactored to read latest executable file from ListOfTests.txt rather than take as CLI 
# 2021/11/05:   T. Endreny, Updated directions, clarified variable names, refactored to use single input path, removed use with Hydro GUI, reformatted all indentations
# 2023/02/19:   T. Endreny, Updated string structure of input paths and parsing of input and output directories to allow more flexibility 
# ========================================================================================================================================================

#import Python libraries
import subprocess
import os
import pandas as pd
import time
from random import uniform
import numpy as np
import csv 
import shutil
import glob
from filecmp import dircmp
#lxml.etree is an XML reader that will preserve comments in the XML
try:
    import lxml.etree as XML_eTree_File
except ImportError:
    print("Warning: The Python XML parsing library 'lxml' is not installed on this machine. It is needed to run this Python script.\n")
    print("Solution: Please install the XML parsing library 'lxml' using one of the following commands, executed from the Anaconda or DOS XTerm with administrator privileges:\n")
    print("conda install -c anaconda lxml")
    print("pip install lxml")
    sys.exit()

#define input and expectedoutput folders containing TestCase files
#Input_Folder_name defined as empty, to accommodate test cases such as multi-quad using quad00# to hold inputs 
#Note: Input_Folder_name is defined below, read from the input string which includes it after the final backslash
folder_expectedoutput = "expectedoutput"
#define ListOfTests_file_name as input ListOfTests.txt
ListOfTests_file_name = "ListOfTests.txt"

#TimeDate_StartProcessingSimulation_seconds is timer at start of simulation
TimeDate_StartProcessingSimulation_seconds = time.time()


#modify_params function called from below to update HydroPlusConfig.xml
#   Values passed for variable names path_to_TestCase_HydroPlusConfgXml, Xml_Key_Elements_String, and path_to_Testing_OuputFolder
def modify_params(path_to_TestCase_HydroPlusConfgXml, Xml_Key_Elements_String, path_to_Testing_OuputFolder):
	#define XML_eTree_File_Tree as parsed set of keys from HydroPlusConfig.xml file
    #Note: This parse function drops all comments in the HydroPlusConfig.xml file
    #Note: This loss then propagates to TestCases expectedresults folders after using z_Copy_TestCaseOutput_to_ExpectedOutput.py
    XML_eTree_File_Tree = XML_eTree_File.parse(path_to_TestCase_HydroPlusConfgXml)
	#define XML_eTree_File_Element the value or path of the Xml_Key_Elements_String <OutputFolder_Path> within HydroPlusConfig.xml file
    XML_eTree_File_Element = XML_eTree_File_Tree.find(Xml_Key_Elements_String)
	#redefine XML_eTree_File_Element to new value, path_to_Testing_OuputFolder, within HydroPlusConfg.xml file
    XML_eTree_File_Element.text = str(path_to_Testing_OuputFolder)
	#write XML_eTree_File_Tree as new HydroPlusConfig.xml file in folder defined by path_to_TestCase_HydroPlusConfgXml
    XML_eTree_File_Tree.write(path_to_TestCase_HydroPlusConfgXml)

#compare_files_completely created to provide use efficient dircmp function
from filecmp import dircmp
import os

#compare_files_completely for strict Testing_ExpectedOutput_path against those in Testing_Output_path using dircmp
#Note: Returns a dictionary with matching, differing, and missing files
def compare_files_completely(Testing_Output_path, Testing_ExpectedOutput_path):
    # Initialize arrays for matching_files, differing_files, missing_files
    matching_files = []
    differing_files = []
    missing_files = []
    
    # Get lists of files in both directories
    expected_files = [f for f in os.listdir(Testing_ExpectedOutput_path) if os.path.isfile(os.path.join(Testing_ExpectedOutput_path, f))]
    output_files = [f for f in os.listdir(Testing_Output_path) if os.path.isfile(os.path.join(Testing_Output_path, f))]

    # Check for missing files in the output folder
    for file in expected_files:
        if file not in output_files:
            missing_files.append(file)  # Add the missing file name

    # Perform dircmp comparison for differing files (optional)
    if os.path.exists(Testing_Output_path) and os.path.exists(Testing_ExpectedOutput_path):
        comparison = dircmp(Testing_Output_path, Testing_ExpectedOutput_path)
        differing_files.extend(comparison.diff_files)

    # Return lists
    return {
        'matching_files': matching_files,
        'differing_files': differing_files,
        'missing_files': missing_files
    }


#compare_files_conditionally for relaxed Testing_ExpectedOutput_path against those in Testing_Output_path using dircmp
#Note: Skip lines containing 'Program Version' if 'CoolBuilding' is in TestCase_Folder_path.
#Note: Returns a dictionary with matching, differing, and missing files
def compare_files_conditionally(Testing_Output_path, Testing_ExpectedOutput_path, TestCase_Folder_path):
    #Initialize arrays for matching_files, differing_files, missing_files
    matching_files = []
    differing_files = []
    missing_files = []

    #skip_lines_containing uses Python ternary conditional operator
    #Note: Trying to avoid comparing files that contain a date of the model run, which EnergyPlus puts in certain output files
    #Note: If the condition is True (i.e., "CoolBuilding" is in TestCase_Folder_path), assign "Program Version" to skip_lines_containing
    #Note: If the condition is False, assign None to skip_lines_containing
    skip_lines_containing = "Program Version" if "CoolBuilding" in TestCase_Folder_path else None

    #expected_files is obtained as list files in Testing_ExpectedOutput_path
    expected_files = os.listdir(Testing_ExpectedOutput_path)

    #for file_name in expected_files
    for file_name in expected_files:
        #expected_file_path is from os.path.join with Testing_ExpectedOutput_path, file_name
        expected_file_path = os.path.join(Testing_ExpectedOutput_path, file_name)
        #output_file_path is from os.path.join with Testing_Output_path, file_name
        output_file_path = os.path.join(Testing_Output_path, file_name)

        #If not os.path.exists(output_file_path) checks if file exists
        #Note: The not keyword negates the result of the condition, so it is True if the file does not exist, allowing it to append missing file_name to list
        if not os.path.exists(output_file_path):
            #missing_files list appended
            missing_files.append(file_name)
            #continue skips further processing for this file
            continue

        #try compare the files with conditional line skipping
        try:
            #with open both expected_file_path output_file_path as files to compare, name them file_expected and file_actual
            with open(expected_file_path, 'r') as file_expected, open(output_file_path, 'r') as file_actual:
                #lines_expected contains all lines from file_expected
                lines_expected = file_expected.readlines()
                #lines_actual contains all lines from file_actual
                lines_actual = file_actual.readlines()

                #If skip_lines_containing is true (it has strings of keywords to avoide) then skip lines containing "Program Version"
                #Note: Trying to avoid comparing files that contain a date of the model run
                if skip_lines_containing:
                    #lines_expected redfined as line in lines_expected if value in skip_lines_containing is not in line 
                    lines_expected = [line for line in lines_expected if skip_lines_containing not in line]
                    #lines_actual redfined as line in lines_actual if value in skip_lines_containing is not in line 
                    lines_actual = [line for line in lines_actual if skip_lines_containing not in line]

                #if lines_expected does not equal lines_actual then 
                if lines_expected != lines_actual:
                    #differing_files is appended with file_name
                    differing_files.append(file_name)
                #Else
                else:
                    #matching_files is appended with file_name
                    matching_files.append(file_name)
                    
        except Exception as e:
            print(f"Error comparing {expected_file_path} and {output_file_path}: {e}")

    #num_files_expected is len of number of files in Testing_ExpectedOutput_path
    num_files_expected = len([f for f in os.listdir(Testing_ExpectedOutput_path) if os.path.isfile(os.path.join(Testing_ExpectedOutput_path, f))])
    #num_files_output is len of number of files in Testing_Output_path
    num_files_output = len([f for f in os.listdir(Testing_Output_path) if os.path.isfile(os.path.join(Testing_Output_path, f))])
    #If num_files_expected less than num_files_output then the user needs notification
    if num_files_expected < num_files_output and num_files_expected > 0:
        #missing_files appends warning 
        missing_files.append("Warning: ExpectedOutput Folder Has Fewer Files Than Output Folder")

    #return lists
    return {
        'matching_files': matching_files,
        'differing_files': differing_files,
        'missing_files': missing_files
    }
    
#open ListOfTests_file_name as ListOfTests_file
with open(ListOfTests_file_name) as ListOfTests_file:
	#define data for each line in ListOfTests_file with tuple command, reading objects separated by commas
    data=[tuple(line) for line in csv.reader(ListOfTests_file)]
	
#define Testing_DateTimeFolder_name folder as Testing_%Y_%m_%d_%H_%M_%S, with variables for time and date components given by strftime() method
Testing_DateTimeFolder_name = time.strftime("Testing_%Y_%m_%d_%H_%M_%S")

#define ListsOfTests_Results_file_name as TestResults.txt within Testing_DateTimeFolder_name folder
ListsOfTests_Results_file_name = Testing_DateTimeFolder_name + "\\TestResults.txt"

#define ListsOfTests_Results_string as strings to keep track of results
ListsOfTests_Results_string = {"Overall Test Results": "Passed"}

#for each row of data in ListOfTests.txt file
for ListOfTests_Input_Row in data:
    #if 1st element of row is Executable key word, then read in path to HydroPlus.exe file
    if ListOfTests_Input_Row[0] == "Executable":
        #name HydroPlus executable
        HydroPlusExe_path = ListOfTests_Input_Row[1]
        #Continue to next row of data, and return to for loop 
        continue
    #if 1st element of row is TestName key word, then this row is the Header row above the TestCase folder, HydroPlus key word, and Testing folder data
    if ListOfTests_Input_Row[0] == "TestName":
        #Continue to next row of data
        continue
        
    #define Testing_CaseFolder_name within as 1st variable in data ListOfTests_Input_Row, counting starts at 0
    Testing_CaseFolder_name = ListOfTests_Input_Row[0]

    #define TestCase_Folder_path as value of 2nd element in ListOfTests_Input_Row, counting starts at 0
    TestCase_Folder_path = ListOfTests_Input_Row[1]

    #Algorithm to parse TestCase_Folder_path for use with input and expectedoutput folders
    #Find final backslash in TestCase_Folder_path string, which is point where input folder is listed
    str_backslash = "\\"
    #TestCase_Folder_w_Input_path defined as 2nd input ListOfTests_Input_Row, TestCase_Folder_path
    TestCase_Folder_w_Input_path = TestCase_Folder_path
    #len_backslash_inputs is length to backslash in TestCase_Folder_w_Input_path using .rfind function
    len_backslash_inputs = TestCase_Folder_w_Input_path.rfind(str_backslash)
    #len_TestCase_Folder_w_Input_path is length of string TestCase_Folder_w_Input_path
    len_TestCase_Folder_w_Input_path = len(TestCase_Folder_w_Input_path)
    #len_Input_Folder_name is length of string input folder
    len_Input_Folder_name = (len_TestCase_Folder_w_Input_path - len_backslash_inputs)
    #len_TestCase_Folder_path is len_backslash_inputs plus 1
    len_TestCase_Folder_path = len_backslash_inputs + 1
    #Input_Folder_name read from right in TestCase_Folder_w_Input_path distance -len_Input_Folder_name:
    Input_Folder_name = TestCase_Folder_w_Input_path[-len_Input_Folder_name:]
    #TestCase_Folder_path read from left in TestCase_Folder_path distance len_TestCase_Folder_path
    TestCase_Folder_path = TestCase_Folder_path[:len_TestCase_Folder_path]
    
    #print("TestCase_Folder_w_Input_path is {}, final folder at {}".format(TestCase_Folder_w_Input_path, len_backslash_inputs))
    #print("Input_Folder_name is {}, len_TestCase_Folder_w_Input_path length {}".format(Input_Folder_name, len_TestCase_Folder_w_Input_path))
    #print("TestCase_Folder_path is {}".format(TestCase_Folder_path))

    #define Testing_DateTimeFolder_CaseFolder_names as combined string Testing_DateTimeFolder_name folder and Testing_CaseFolder_name, the unique test case name 
    Testing_DateTimeFolder_CaseFolder_names = time.strftime(Testing_DateTimeFolder_name + "\\" + Testing_CaseFolder_name)
    #define Testing_Output_path within Testing_DateTimeFolder_CaseFolder_names folder with name for output
    #Note: When a module is loaded from a file in Python, __file__ is set to its path.
    Testing_Output_path = os.path.join(os.path.dirname(__file__),Testing_DateTimeFolder_CaseFolder_names + '\\output')
    #define Testing_ExpectedOutput_path within Testing_DateTimeFolder_CaseFolder_names folder with name for expectedoutput
    Testing_ExpectedOutput_path = os.path.join(os.path.dirname(__file__),Testing_DateTimeFolder_CaseFolder_names + '\\expectedoutput')
    #define Testing_Input_path within Testing_DateTimeFolder_CaseFolder_names folder with name for input
    Testing_Input_path = os.path.join(os.path.dirname(__file__),Testing_DateTimeFolder_CaseFolder_names + '\\input')
     
    #make Testing_Output_path if it does not exit
    if not os.path.exists(Testing_Output_path):
        os.makedirs(Testing_Output_path)
    #make Testing_Input_path if it does not exit
    if not os.path.exists(Testing_Input_path):
        os.makedirs(Testing_Input_path)
    #make Testing_ExpectedOutput_path if it does not exit
    if not os.path.exists(Testing_ExpectedOutput_path):
        os.makedirs(Testing_ExpectedOutput_path)
		
    #define TestCase_Input_path as input path, 2nd variable in data ListOfTests_Input_Row, as upper case string
    TestCase_Input_path = TestCase_Folder_path + Input_Folder_name
        
    #for each file in the input path +/*.*, finding all file paths with glob command
    for file in glob.glob(TestCase_Input_path+'/*.*'):
        #copy file to Testing_Input_path directory
        shutil.copy(file, Testing_Input_path)

    #define TestCase_ExpectedOutput_path as expectedoutput path, 3nd variable in data ListOfTests_Input_Row, as upper case string; counter starts at 0
    TestCase_ExpectedOutput_path = TestCase_Folder_path + folder_expectedoutput
    
    #for each file in the expectedoutput path +/*.*, finding all file paths with glob command
    for file in glob.glob(TestCase_ExpectedOutput_path+'/*.*'):
        #copy file to newinpath directory
        shutil.copy(file, Testing_ExpectedOutput_path)

    #copy TestCase HydroPlusConfig.xml to Testing folder
    #shutil.copy function transfers elements on left to elements on right within {} 
    shutil.copy(TestCase_Input_path + '\\HydroPlusConfig.xml', Testing_Input_path + '\\HydroPlusConfig.xml')

    #-------------------------------------------------------------------------------------------------------------------------
    #Replace elements within HydroPlusConfig.xml file to swap out preserved TestCase with the new Testing case
    #-------------------------------------------------------------------------------------------------------------------------
    #call modify_params function to give path for Testing input directory and output path within HydroPlusConfig.xml file 
    modify_params(Testing_Input_path+"\\HydroPlusConfig.xml","SimulationStringParams/OutputFolder_Path", Testing_Output_path+"\\")

    #-------------------------------------------------------------------------------------------------------------------------
    #Command Line Input to call HydroPlus.exe using Python function subprocess.call
    #-------------------------------------------------------------------------------------------------------------------------
    subprocess.call([HydroPlusExe_path,Testing_Input_path])
    #print the HydroPlus command line argument
    print('Finished test for: {}\n\n'.format(Testing_Input_path))

   
    #pause program for 1 sec, to complete above call
    time.sleep(1)
        
    #-------------------------------------------------------------------------------------------------------------------------
    #Conduct test of differences between expected outputs vs outputs 
    #-------------------------------------------------------------------------------------------------------------------------
    #If CoolBuilding is in TestCase_Folder_path then use compare_files_conditionally for file comparison to avoid comparing time stamps
    if ('CoolBuilding' in TestCase_Folder_path):
        #Expected_vs_Output_Difference_Test_Result is three lists of file comparison results from compare_files_conditionally
        Expected_vs_Output_Difference_Test_Result = compare_files_conditionally(Testing_Output_path, Testing_ExpectedOutput_path, TestCase_Folder_path)
    #Else use compare_files_completely function which is faster, using Python dircmp 
    else:
        #Expected_vs_Output_Difference_Test_Result is three lists of file comparison results from compare_files_completely
        Expected_vs_Output_Difference_Test_Result = compare_files_completely(Testing_Output_path, Testing_ExpectedOutput_path)

    #If Not TestCase_ExpectedOutput_path True then it is empty and a Warning is needed
    #Note: TestCase_ExpectedOutput_path should contain files to compare, but if it is empty there is no default warning, so this notifies the user
    if not os.listdir(TestCase_ExpectedOutput_path):
        # Add warning to missing_files
        Expected_vs_Output_Difference_Test_Result['missing_files'].append("Warning: ExpectedOutput Folder Has No Files")
        ListsOfTests_Results_string["Overall Test Results"] = "Failed"
        ListsOfTests_Results_string[ListOfTests_Input_Row[0]] = "Failed"

    #If Not Testing_Output_path True then it is empty and a Warning is needed
    #Note: Testing_Output_path should contain files to compare, but if it is empty there is no default warning, so this notifies the user
    if not os.listdir(Testing_Output_path):
        # Add warning to missing_files
        Expected_vs_Output_Difference_Test_Result['missing_files'].append("Warning: Output Folder Has No Files")
        ListsOfTests_Results_string["Overall Test Results"] = "Failed"
        ListsOfTests_Results_string[ListOfTests_Input_Row[0]] = "Failed"

    #-------------------------------------------------------------------------------------------------------------------------
    # Write test result file
    #-------------------------------------------------------------------------------------------------------------------------
    with open(ListsOfTests_Results_file_name, 'a') as ListsOfTests_Results_file:

        # Write to string the folder name and test result = Passed
        ListsOfTests_Results_string[ListOfTests_Input_Row[0]] = "Passed"

        # Write folder and path details to the test results file
        ListsOfTests_Results_file.write("Test Folder name: " + Testing_CaseFolder_name + "\n")
        ListsOfTests_Results_file.write("Path of new output: " + Testing_Output_path + "\n")
        ListsOfTests_Results_file.write("Path of expected output: " + Testing_ExpectedOutput_path + "\n")
            
        #-------------------------------------------------------------------------------------------------------------------------
        # Handle missing files
        #-------------------------------------------------------------------------------------------------------------------------
        counter_missing = 0
        #For loop only initiates if there are files in Expected_vs_Output_Difference_Test_Result['missing_files']
        for Expected_vs_Output_Missing_file in Expected_vs_Output_Difference_Test_Result['missing_files']:
            #If counter_missing is zero at first pass because there are files in Expected_vs_Output_Difference_Test_Result['missing_files']
            if counter_missing == 0:
                # Announce missing files
                ListsOfTests_Results_file.write("\tTest case failed due to the following expected output files missing from the new output:\n")
            
            # Write each missing file
            ListsOfTests_Results_file.write("\t\t" + Expected_vs_Output_Missing_file + "\n")
            ListsOfTests_Results_string["Overall Test Results"] = "Failed"
            ListsOfTests_Results_string[ListOfTests_Input_Row[0]] = "Failed"
            counter_missing += 1

        #-------------------------------------------------------------------------------------------------------------------------
        # Handle differing files
        #-------------------------------------------------------------------------------------------------------------------------
        if len(Expected_vs_Output_Difference_Test_Result['differing_files']) > 0:
            ListsOfTests_Results_file.write("\tTest case failed due to the following output files containing different content from expected output:\n")
            ListsOfTests_Results_string["Overall Test Results"] = "Failed"
            ListsOfTests_Results_string[ListOfTests_Input_Row[0]] = "Failed"

            # Write each differing file
            for diff in Expected_vs_Output_Difference_Test_Result['differing_files']:
                ListsOfTests_Results_file.write("\t\t" + diff + "\n")
                     
        ListsOfTests_Results_file.write("==============================================================================\n\n")
        #close ListsOfTests_Results_file_name
        ListsOfTests_Results_file.close()
		 
#print to screen out ListsOfTests_Results_string
print('\n\nSummary of Tests: {}.'.format(ListsOfTests_Results_string))
print('\nReview file ..\\{} for details.'.format(ListsOfTests_Results_file_name))

#TimeDate_StopProcessingSimulation_seconds (s) is clock time when simulation processing ends
TimeDate_StopProcessingSimulation_seconds = time.time()
#Time_DurationToProcessSimulation_seconds (s) is difference between start and stop of processing simulation
Time_DurationToProcessSimulation_seconds = TimeDate_StopProcessingSimulation_seconds - TimeDate_StartProcessingSimulation_seconds
#Print out simulation runtime
print("\n" + "TestCases runtime was: {:.1f} minutes".format(Time_DurationToProcessSimulation_seconds/60))

#Write out the exe modified time
DateModified = TimeDate_StartProcessingSimulation_seconds - os.path.getmtime(HydroPlusExe_path)
print ("HydroPlus.exe file used in test cases has vintage: {:.1f} minutes, or {:.1f} days.".format(DateModified/60, DateModified/60/60/24))

#open ListsOfTests_Results_file_name TestResults.txt as ListsOfTests_Results_file
with open(ListsOfTests_Results_file_name,'a') as ListsOfTests_Results_file:
    #for each ListOfTests_Input_Row result in ListsOfTests_Results_string
    for TestResults_file in ListsOfTests_Results_string:

        #write result of ListOfTests_Input_Row
        ListsOfTests_Results_file.write(TestResults_file + ":\t" + str(ListsOfTests_Results_string[TestResults_file]))
        ListsOfTests_Results_file.write("\n")

ListsOfTests_Results_file.close()

