
package com.engineous.system.drm.ldlev;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.PrintStream;
import java.io.StringReader;
import java.util.Arrays;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.StringTokenizer;

import com.engineous.common.i18n.IString;
import com.engineous.common.io.IOUtil;
import com.engineous.sdk.log.Log;
import com.engineous.sdk.log.SysLog;
import com.engineous.sdk.pse.DRMCommandRunner;
import com.engineous.sdk.pse.DRMEnabler;
import com.engineous.sdk.pse.DRMJobID;
import com.engineous.sdk.pse.DRMJobStatus;
import com.engineous.sdk.pse.DRMJobValue;
import com.engineous.sdk.pse.DRMResourceRequest;
import com.engineous.sdk.pse.DRMHostInfo;
import com.engineous.sdk.pse.PSEException;
import com.engineous.sdk.pse.PSENonRetryableException;
import com.engineous.sdk.pse.SysFiper;

/**
 * This class implements the protocol which must be implemented for
 * the Simulia Execution Engine to be able to use Load Leveler to
 * evaluate Isight workitems as part of an Isight model execution.
 */
public class LoadLevelerEnabler
		implements DRMEnabler {

	private static final Class<LoadLevelerEnabler> CLASS = LoadLevelerEnabler.class;

	// List of resource types, used to construct property
	// assignment commands in the 'llsubmit' command file:
	private static final String[] LDLEV_RES_KEYS = {
		LoadLevelerUtils.LDLEV_RESOURCE_JOB_NAME,
		LoadLevelerUtils.LDLEV_RESOURCE_CLASS,
		LoadLevelerUtils.LDLEV_RESOURCE_ACCOUNT,
		LoadLevelerUtils.LDLEV_RESOURCE_PARALLEL_TASKS,
		LoadLevelerUtils.LDLEV_RESOURCE_MAX_MEMORY,
		LoadLevelerUtils.LDLEV_RESOURCE_START_DATE,
		LoadLevelerUtils.LDLEV_RESOURCE_RUN_LIMIT,
		LoadLevelerUtils.LDLEV_RESOURCE_WORKING_DIR,
	};

	// List of property names, used to construct property
	// assignment commands in the 'llsubmit' command file:
	private static final String[] LDLEV_RES_PROPS = {
		"job_name",         // LoadLevelerUtils.LDLEV_RESOURCE_JOB_NAME,
		"class",            // LoadLevelerUtils.LDLEV_RESOURCE_CLASS,
		"account_no",       // LoadLevelerUtils.LDLEV_RESOURCE_ACCOUNT,
		"total_tasks",      // LoadLevelerUtils.LDLEV_RESOURCE_PARALLEL_TASKS,
		"resources",        // LoadLevelerUtils.LDLEV_RESOURCE_MAX_MEMORY,
		"startdate",        // LoadLevelerUtils.LDLEV_RESOURCE_START_DATE,
		"wall_clock_limit", // LoadLevelerUtils.LDLEV_RESOURCE_RUN_LIMIT,
		"initialdir",       // LoadLevelerUtils.LDLEV_RESOURCE_WORKING_DIR,
	};

	// For every job submitted to Load Leveler, the output
	// of the 'llsubmit' command is a String containing the
	// Load Leveler job ID prefixed by this string [NOTE:
	// the job ID is also followed by a matching '"']:
	private static final String llSubmitJobIDPrefix = "llsubmit: The job \"";

	// Complete list of Load Leveler job status values, with
	// corresponding DRM-Enabler status values noted [NOTE:
	// values are kept sorted for quick translation]:
	private static final String[] ldlevStatuses = {
		"C",   // DRMJobStatus.DRM_JOBSTATUS_DONE
		"CA",  // DRMJobStatus.DRM_JOBSTATUS_DONE
		"CK",  // DRMJobStatus.DRM_JOBSTATUS_RUNNING
		"CP",  // DRMJobStatus.DRM_JOBSTATUS_DONE
		"D",   // DRMJobStatus.DRM_JOBSTATUS_PENDING
		"E",   // DRMJobStatus.DRM_JOBSTATUS_PAUSED
		"EP",  // DRMJobStatus.DRM_JOBSTATUS_PAUSED
		"H",   // DRMJobStatus.DRM_JOBSTATUS_PENDING
		"HS",  // DRMJobStatus.DRM_JOBSTATUS_PENDING
		"I",   // DRMJobStatus.DRM_JOBSTATUS_PENDING
		"MP",  // DRMJobStatus.DRM_JOBSTATUS_PAUSED
		"NQ",  // DRMJobStatus.DRM_JOBSTATUS_PENDING
		"NR",  // DRMJobStatus.DRM_JOBSTATUS_PENDING
		"P",   // DRMJobStatus.DRM_JOBSTATUS_PENDING
		"R",   // DRMJobStatus.DRM_JOBSTATUS_RUNNING
		"RM",  // DRMJobStatus.DRM_JOBSTATUS_DONE
		"RP",  // DRMJobStatus.DRM_JOBSTATUS_DONE
		"S",   // DRMJobStatus.DRM_JOBSTATUS_PENDING
		"ST",  // DRMJobStatus.DRM_JOBSTATUS_PENDING
		"TX",  // DRMJobStatus.DRM_JOBSTATUS_DONE
		"V",   // DRMJobStatus.DRM_JOBSTATUS_PENDING
		"VP",  // DRMJobStatus.DRM_JOBSTATUS_PENDING
		"X",   // DRMJobStatus.DRM_JOBSTATUS_PENDING
		"XP",  // DRMJobStatus.DRM_JOBSTATUS_PENDING
	};

	// Complete list of DRM-Enabler job status values,
	// with corresponding Load Leveler status values
	// noted:
	private static final int[] drmStatuses = {
		DRMJobStatus.DRM_JOBSTATUS_DONE,     // C
		DRMJobStatus.DRM_JOBSTATUS_DONE,     // CA
		DRMJobStatus.DRM_JOBSTATUS_RUNNING,  // CK
		DRMJobStatus.DRM_JOBSTATUS_DONE,     // CP
		DRMJobStatus.DRM_JOBSTATUS_PENDING,  // D
		DRMJobStatus.DRM_JOBSTATUS_PAUSED,   // E
		DRMJobStatus.DRM_JOBSTATUS_PAUSED,   // EP
		DRMJobStatus.DRM_JOBSTATUS_PENDING,  // H
		DRMJobStatus.DRM_JOBSTATUS_PENDING,  // HS
		DRMJobStatus.DRM_JOBSTATUS_PENDING,  // I
		DRMJobStatus.DRM_JOBSTATUS_PAUSED,   // MP
		DRMJobStatus.DRM_JOBSTATUS_PENDING,  // NQ
		DRMJobStatus.DRM_JOBSTATUS_PENDING,  // NR
		DRMJobStatus.DRM_JOBSTATUS_PENDING,  // P
		DRMJobStatus.DRM_JOBSTATUS_RUNNING,  // R
		DRMJobStatus.DRM_JOBSTATUS_DONE,     // RM
		DRMJobStatus.DRM_JOBSTATUS_DONE,     // RP
		DRMJobStatus.DRM_JOBSTATUS_PENDING,  // S
		DRMJobStatus.DRM_JOBSTATUS_PENDING,  // ST
		DRMJobStatus.DRM_JOBSTATUS_DONE,     // TX
		DRMJobStatus.DRM_JOBSTATUS_PENDING,  // V
		DRMJobStatus.DRM_JOBSTATUS_PENDING,  // VP
		DRMJobStatus.DRM_JOBSTATUS_PENDING,  // X
		DRMJobStatus.DRM_JOBSTATUS_PENDING,  // XP
	};

	// THESE VALUES MAY BE OVERRIDDEN BY SYSTEM PROPERTIES
	// WHEN A LoadLevelerEnabler INSTANCE IS CONSTRUCTED:
	private String llSubmitCmd = "llsubmit";
	private String llStationCommand = "transtation";
	private String llSubmitStdoutFile = "/dev/null";
	private String llSubmitStderrFile = "/dev/null";
	private boolean llSaveSubmitFile = false;
	private String llSubmitResultFile = null;

	private String llCancelCmd = "llcancel";

	private String llQueryCmd = "llq";
	private int llQueryChunkSize = 50;

	private String  llStatusCmd = "llstatus";

	private long llCmdWaitTime = 60000;  // DEFAULT 60 SECONDS (in millisecond units)

	/**
	 * Constructor for LoadLevelerEnabler objects.  All DRM Enabler
	 * classes are packaged as 'Runtime' classes in DRM metamodels,
	 * and so are required to have a default constructor.
	 */
	public LoadLevelerEnabler() {

		// DETECT AND SAVE THE ADJUSTABLE LOAD LEVELER COMMANDS AND ARGUMENTS:
		String llSubmitCmd = SysFiper.getFiperConfig().getProperty("fiper.system.ldlevSubmitPath");
		if ((llSubmitCmd != null) && (llSubmitCmd.length() > 0)) {
			this.llSubmitCmd = llSubmitCmd;
		}

		String llStationCommand = SysFiper.getFiperConfig().getProperty("fiper.system.ldlevStationCommand");
		if ((llStationCommand != null) && (llStationCommand.length() > 0)) {
			this.llStationCommand = llStationCommand;
		}

		String llSubmitStdoutFile = SysFiper.getFiperConfig().getProperty("fiper.system.ldlevSubmitStdout");
		if ((llSubmitStdoutFile != null) && (llSubmitStdoutFile.length() > 0)) {
			this.llSubmitStdoutFile = llSubmitStdoutFile;
		}

		String llSubmitStderrFile = SysFiper.getFiperConfig().getProperty("fiper.system.ldlevSubmitStderr");
		if ((llSubmitStderrFile != null) && (llSubmitStderrFile.length() > 0)) {
			this.llSubmitStderrFile = llSubmitStderrFile;
		}

		String llSubmitResultFile = SysFiper.getFiperConfig().getProperty("fiper.system.ldlevSubmitResult");
		if ((llSubmitResultFile != null) && (llSubmitResultFile.length() > 0)) {
			this.llSubmitResultFile = llSubmitResultFile;
		}

		String llCancelCmd = SysFiper.getFiperConfig().getProperty("fiper.system.ldlevCancelPath");
		if ((llCancelCmd != null) && (llCancelCmd.length() > 0)) {
			this.llCancelCmd = llCancelCmd;
		}

		String llQueryCmd = SysFiper.getFiperConfig().getProperty("fiper.system.ldlevQueryPath");
		if ((llQueryCmd != null) && (llQueryCmd.length() > 0)) {
			this.llQueryCmd = llQueryCmd;
		}

		try {
			this.llQueryChunkSize = Integer.parseInt(SysFiper.getFiperConfig().getProperty("fiper.system.ldlevQueryChunkSize", "50"));
		}
		catch (NumberFormatException e1) {
			SysLog.logWarn(new IString(CLASS, 0, "Illegal chunk size in fiper.system.ldlevQueryChunkSize: {0}.", e1.getMessage()));
		}

		String llStatusCmd = SysFiper.getFiperConfig().getProperty("fiper.system.ldlevStatusPath");
		if ((llStatusCmd != null) && (llStatusCmd.length() > 0)) {
			this.llStatusCmd = llStatusCmd;
		}

		String llCmdWaitTimeStr = SysFiper.getFiperConfig().getProperty("fiper.system.ldlevCmdWaitTime", "60");
		try {
			llCmdWaitTime = Integer.parseInt(llCmdWaitTimeStr) * 1000;    // Wait time in ms
		}
		catch (NumberFormatException e2) { // If string can't be parsed, use default
			SysLog.logWarn(new IString(CLASS, 0, "Illegal Load Leveler wait time in fiper.system.ldlevCmdWaitTime: {0}.", e2.getMessage()));
		}

		llSaveSubmitFile = Boolean.parseBoolean(SysFiper.getFiperConfig().getProperty("fiper.system.ldlevSaveSubmitFile", "false"));
	}

	/**
	 * The Isight job database requires that each workitem DB record
	 * identify the DRM job, if any, which is managing that workitem.
	 * The workitem DB table records the DRM system's job ID in a
	 * string column; therefore DRM job IDs must be strings, or at
	 * least must be interconvertible with a string format.  This
	 * method converts string from the workitem record to a job ID.
	 * @param jobID String
	 * @return DRMJobID
	 */
	public DRMJobID makeDRMJobIDFromString(String jobID) {
		return new LoadLevelerJobID(jobID);
	}

	/**
	 * This method is called by the Simulia Execution Engine to create
	 * a new Load Leveler job, providing it with all data needed to
	 * evaluate a given Isight workitem [by launching an Isight Station
	 * and telling it to claim that particular workitem].
	 * @param jobID         String
	 * @param workItemID    String
	 * @param genResReq     DRMResourceRequest
	 * @param drmResReq     String
	 * @param submitHost    DRMHostInfo
	 * @param commandRunner DRMCommandRunner
	 * @return DRMJobID
	 * @throws PSEException
	 */
	public DRMJobID submit(String jobID, String workItemID, DRMResourceRequest genResReq, String drmResReq, DRMHostInfo submitHost, DRMCommandRunner commandRunner)
			throws PSEException {

		String[] cmd = buildLoadLevelerDispatchCommand(workItemID,LoadLevelerUtils.fromString(drmResReq));
		return runLLSubmit(cmd,commandRunner);
	}

	/**
	 * This method is called by the Simulia Execution Engine to create
	 * a new Load Leveler job, providing it with all data needed to
	 * 'co-evaluate' a given set of Isight workitems [by launching an
	 * Isight Station and telling it to claim that particular set of
	 * workitem].  It has been declared separately from 'submit' spe-
	 * cifically to allow DRM Enablers to support the Isight 'cosimu-
	 * lation' feature.
	 * @param jobID         String
	 * @param cosimGroupID  String
	 * @param workItemIDSet List<String>
	 * @param cpuCountSet   List<Integer>
	 * @param genResReq     DRMResourceRequest
	 * @param drmResReqSet  List<String>
	 * @param submitHost    DRMHostInfo
	 * @param commandRunner DRMCommandRunner
	 * @throws PSEException
	 */
	public DRMJobID cosubmit(String jobID, String cosimGroupID, List<String> workItemIDSet, DRMResourceRequest genResReq, List<String> drmResReqSet, DRMHostInfo submitHost, DRMCommandRunner commandRunner)
			throws PSEException {

		throw new PSENonRetryableException(new IString(CLASS, 0, "Load Leveler doesn't implement 'co-submit' (yet?)."));
	}

	/**
	 * This method constructs and returns an 'llsubmit' command which
	 * launches a Load Leveler job that runs one or more SEE workitems in
	 * a Station launched by Load Leveler.  Both regular and 'cosimulation'
	 * jobs are handled.  Generic and Load Leveler-specific resource sett-
	 * ings are incorporated into the command as 'llsubmit' arguments.
	 * <p>
	 * This method has been factored out so that it may be invoked by the
	 * Load Leveler unit tests which verify the command construction
	 * procedure.
	 * @param workItemIDArg     String
	 * @param drmResReq         Map<String,String>
	 * @param commandRunner     DRMCommandRunner
	 * @return String[]
	 * @throws PSEException
	 */
	private String[] buildLoadLevelerDispatchCommand(String workItemIDArg, Map<String,String> drmResReq)
			throws PSEException {

		final String[] cmd = { llSubmitCmd, "" };

		// Create LoadLeveler command file
		FileOutputStream fOut = null;
		PrintStream ps = null;
		PSEException pseex = null;

		try {
			final File cmdFile = new File(IOUtil.getLocalTempDirectory(), workItemIDArg + ".cmd");
			cmd[1] = cmdFile.getAbsolutePath();

			// Print commands to the file
			fOut = new FileOutputStream(cmdFile);
			ps = new PrintStream(fOut);

			// Write commands to file
			ps.println("# @ input = /dev/null");
			ps.println("# @ environment = EUG_MPI=1;MP_INFOLEVEL=2");
			ps.println("# @ job_type = serial");
			ps.println("# @ output = " + llSubmitStdoutFile);
			ps.println("# @ error = " + llSubmitStderrFile);

			// Write resource-request commands to file:
			if (drmResReq != null) {
				// PREPROCESSING #1: If no job name specified,
				// put a default expression into the table:
				String jobName = drmResReq.get(LoadLevelerUtils.LDLEV_RESOURCE_JOB_NAME);
				if (jobName.length() < 1) {
					drmResReq.put(LoadLevelerUtils.LDLEV_RESOURCE_JOB_NAME,"Fiper_DRM.$(jobid)");
				}

				// PREPROCESSING #2: If max memory value IS specified,
				// wrap the value as required by 'llsubmit' and put
				// the result into the table:
				String maxMem = drmResReq.get(LoadLevelerUtils.LDLEV_RESOURCE_MAX_MEMORY);
				if (maxMem.length() > 0) {
					drmResReq.put(LoadLevelerUtils.LDLEV_RESOURCE_MAX_MEMORY,"ConsumableMemory(" + maxMem + ")");
				}

				// PROCESS ALL RESOURCE REQUESTS:
				for (int i=0; i<LDLEV_RES_KEYS.length; i++) {
					String resValue = drmResReq.get(LDLEV_RES_KEYS[i]);
					if (resValue.length() > 0) {
						ps.println("# @ " + LDLEV_RES_PROPS[i] + " = " + resValue);
					}
				}
			}
			else {
				// ALWAYS mark as a FIPER job if not overridden by user:
				// [note: this case will occur only if this method is
				//        called separately - e.g by the unit test!]
				ps.println("# @ job_name = Fiper_DRM.$(jobid)");
			}

			// Finish the command file:
			ps.println("# @ queue");
			ps.println(llStationCommand + " ID:" + workItemIDArg);
			ps.println("exit $?");
		}
		catch (IOException ioe) {
			pseex = new PSENonRetryableException(ioe, new IString(CLASS, 0, "Unable to create Load Leveler command file"));
		}
		finally {
			if (fOut != null) {
				try {
					fOut.close();
				}
				catch (IOException ioe) {
					if (pseex == null) {
						pseex = new PSENonRetryableException(ioe, new IString(CLASS, 0, "Unable to close Load Leveler command file"));
					}
				}
			}
		}

		if (pseex != null) {
			throw pseex;
		}

		return cmd;
	}

	/**
	 * This method runs the given command via the given DRM command
	 * runner.  The command output is parsed to extract the Load
	 * Leveler job ID, which is returned packaged as a DRMJobID
	 * object.
	 * <p>
	 * NOTE: The 'llsubmit' command doesn't take direct arguments,
	 * it takes the name of a command file as its only argument.
	 * This is a temporary file which must be deleted after use.
	 * @param cmd           String[]
	 * @param commandRunner DRMCommandRunner
	 * @return DRMJobID
	 * @throws PSEException
	 */
	private DRMJobID runLLSubmit(String[] cmd, DRMCommandRunner commandRunner)
			throws PSEException {

		try {
			String llJobIDStr = "unknown";
			String stdOutStr = commandRunner.run(cmd, "", llCmdWaitTime);

			// Parse the llsubmit stdOut for the Load Leveler
			// Job ID to store in the database:
			int start = stdOutStr.indexOf(llSubmitJobIDPrefix);
			if (start >= 0) {
				// Move to the quote character in the string with the submission job ID
				start = stdOutStr.indexOf('"', start) + 1;
				int stop = stdOutStr.indexOf('"', start);
				if (stop > start) {
					llJobIDStr = stdOutStr.substring(start, stop);				
				}
			}

			SysLog.log(Log.DEBUG, new IString(CLASS, -1, "Load Leveler Job ID = {0}", llJobIDStr ));

			// If so asked, save the 'llsubmit' output to a file:
			if (llSubmitResultFile != null) {
				PrintStream ps = null;
				try {
					ps = new PrintStream(new FileOutputStream(llSubmitResultFile));
					ps.println(stdOutStr);
				}
				catch (IOException ioe) {
					SysLog.log(Log.ERROR, new IString(CLASS, -1, "Error saving 'llsubmit' output to file \"{0}\".", llSubmitResultFile));
				}
				finally {
					if (ps != null) {
						ps.flush();
						ps.close();
					}
				}
			}

			return new LoadLevelerJobID(llJobIDStr);
		}
		finally {
			// Delete the 'llsubmit' command file:
			File f = new File(cmd[1]);
			if ((f != null) && f.exists()) {
				if (llSaveSubmitFile) {
					SysLog.log(Log.INFO, new IString(CLASS, -1, "Load Leveler command file \"{0}\" has been saved.", cmd[1] ));
				}
				else {
					f.delete();
				}
			}
		}
	}

	/**
	 * This method is called by the Simulia Execution Engine to cancel
	 * a set of Load Leveler jobs currently managed by this Load Leveler
	 * system.  ALL of the DRM ID objects in the given list must be
	 * be LoadLevelerJobID objects.
	 * @param drmJobIDList List<DRMJobID>
	 * @param commandRunner DRMCommandRunner
	 * @throws PSEException
	 */
	public void cancel(List<DRMJobID> drmJobIDList, DRMCommandRunner commandRunner)
			throws PSEException {

		// Need to use an explicit iterator
		// in order to easily handle batching
		Iterator<DRMJobID> drmJobIter = drmJobIDList.iterator();
		int nLeft = drmJobIDList.size();
		while (drmJobIter.hasNext()) {

			String[] cmd = new String[Math.min(nLeft, llQueryChunkSize) + 1];
			cmd[0] = llCancelCmd;

			int count = 1;
			while (drmJobIter.hasNext() && (count < cmd.length)) {
				cmd[count++] = drmJobIter.next().getDRMJobIDAsString();
				nLeft -= 1;
			}

			commandRunner.run(cmd, null, 0);  // don't wait for 'llcancel' to finish
		}
	}

	/**
	 * This method is called by the Simulia Execution Engine to check
	 * the statuses of a set of Load Leveler jobs currently managed
	 * by this Load Leveler system.  ALL of the DRM ID objects in the
	 * given list must be LoadLevelerJobID objects.
	 * @param drmJobIDList List<DRMJobID>
	 * @param commandRunner DRMCommandRunner
	 * @return List<DRMJobValue>
	 * @throws PSEException
	 */
	public List<DRMJobValue> query(List<DRMJobID> drmJobIDList, DRMCommandRunner commandRunner)
			throws PSEException {

		List<DRMJobValue> jobStats = new ArrayList<DRMJobValue>();

		if ( !drmJobIDList.isEmpty() ) {
			// Need to use an explicit iterator
			// in order to easily handle batching
			Iterator<DRMJobID> drmJobIter = drmJobIDList.iterator();
			int nLeft = drmJobIDList.size();
			while (drmJobIter.hasNext()) {

				String[] cmd = new String[Math.min(nLeft, llQueryChunkSize) + 4];
				cmd[0] = llQueryCmd;
				cmd[1] = "-r";
				cmd[2] = "%id";
				cmd[3] = "%st";

				int count = 4;
				while (drmJobIter.hasNext() && (count < cmd.length)) {
					cmd[count++] = drmJobIter.next().getDRMJobIDAsString();
					nLeft -= 1;
				}

				String jobResults = commandRunner.run(cmd, "JOBID", llCmdWaitTime);

				if ((jobResults != null) && (jobResults.length() > 0)) {
					BufferedReader reader = new BufferedReader(new StringReader(jobResults));
					try {
						String line = reader.readLine();    // Throw away the first header line
						while ((line = reader.readLine()) != null) {
							StringTokenizer st = new StringTokenizer(line,"!\n\r\f");
							String jobID = st.nextToken();
							String ldlevStatus = st.nextToken();
							int drmStatus = DRMJobStatus.DRM_JOBSTATUS_UNDEFINED;
							int i = Arrays.binarySearch(ldlevStatuses,ldlevStatus,String.CASE_INSENSITIVE_ORDER);
							if (i >= 0) {
								drmStatus = drmStatuses[i];
							}

							jobStats.add(new DRMJobValue(makeDRMJobIDFromString(jobID),drmStatus));
						}
					}
					catch (IOException ex) {
						SysLog.logWarn(new IString(CLASS, 0, "Unable to parse Load Leveler job status: {0}.", ex.getMessage()));
					}
				}
			}
		}

		return jobStats;
	}

	/**
	 * This method is called by the Simulia Execution Engine to create
	 * a String which describes the execution resources provided speci-
	 * fically by this Load Leveler system.  This String may be returned
	 * to Isight for the LoadLevelerOptionsPanel used to select Load
	 * Leveler resources needed to evaluate workitems.
	 * @param commandRunner DRMCommandRunner
	 * @return String
	 * @throws PSEException
	 */
	public String getDRMResourceSet(DRMCommandRunner commandRunner)
			throws PSEException {
		return "";
	}

	/**
	 * This method is called by the Simulia Execution Engine to identify
	 * the host machines managed by this Load Leveler system.  This list
	 * may be returned to Isight so that users may choose which host, if
	 * any, is to run submitted jobs that are associated with specific
	 * workitems.
	 * @param commandRunner DRMCommandRunner
	 * @return List<DRMHostInfo>
	 * @throws PSEException
	 */
	public List<DRMHostInfo> getAvailableHosts(DRMCommandRunner commandRunner)
			throws PSEException {

		List<DRMHostInfo> hosts = new ArrayList<DRMHostInfo>();

		String[] cmd = new String[] { llStatusCmd, "-L", "machine", "-r", "%n", "%o", "%cpu" };
		String hostResults = commandRunner.run(cmd, "", llCmdWaitTime);

		if ((hostResults != null) && (hostResults.length() > 0)) {
			BufferedReader reader = new BufferedReader(new StringReader(hostResults));
			try {
				String line;
				while ((line = reader.readLine()) != null) {
					String[] hostData = line.split("!");
					int numCpu = 1;
					try {
						numCpu = Integer.parseInt(hostData[2]);
					}
					catch (NumberFormatException nfe) {}    // Do the best that we can, default to 1

					hosts.add(new DRMHostInfo(hostData[0],hostData[1],numCpu));
				}
			}
			catch (IOException ex) {
				SysLog.logWarn(new IString(CLASS, 0, "Unable to parse Load Leveler host CPU count: {0}.", ex.getMessage()));
			}
		}

		return hosts;
	}
}
