// // Copyright (C) 2016 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // namespace Google.Android.AppBundle.Editor.Internal.PlayServices { using System; using System.Collections.Generic; using System.Diagnostics; using System.IO; using System.Text; using System.Text.RegularExpressions; using System.Threading; public static class CommandLine { // Constants used for StreamData's handle field. public const int StandardOutputStreamDataHandle = 0; public const int StandardErrorStreamDataHandle = 1; /// /// Result from Run(). /// public class Result { /// String containing the standard output stream of the tool. public string stdout; /// String containing the standard error stream of the tool. public string stderr; /// Exit code returned by the tool when execution is complete. public int exitCode; /// String that can be used in an error message. /// This contains: /// * The command executed. /// * Standard output string. /// * Standard error string. /// * Exit code. public string message; }; /// /// Called when a RunAsync() completes. /// public delegate void CompletionHandler(Result result); /// /// Text and byte representations of an array of data. /// public class StreamData { /// /// Handle to the stream this was read from. /// e.g 0 for stdout, 1 for stderr. /// public int handle = 0; /// /// Text representation of "data". /// public string text = ""; /// /// Array of bytes or "null" if no data is present. /// public byte[] data = null; /// /// Whether this marks the end of the stream. /// public bool end; /// /// Initialize this instance. /// /// Stream identifier. /// String /// Bytes /// Whether this is the end of the stream. public StreamData(int handle, string text, byte[] data, bool end) { this.handle = handle; this.text = text; this.data = data; this.end = end; } /// /// Get an empty StreamData instance. /// public static StreamData Empty { get { return new StreamData(0, "", null, false); } } } /// /// Called when data is received from either the standard output or standard error /// streams with a reference to the current standard input stream to enable simulated /// interactive input. /// /// Executing process. /// Standard input stream. /// Data read from the standard output or error streams. public delegate void IOHandler(Process process, StreamWriter stdin, StreamData streamData); /// /// Asynchronously execute a command line tool, calling the specified delegate on /// completion. /// /// Tool to execute. /// String to pass to the tools' command line. /// Called when the tool completes. /// Directory to execute the tool from. /// Additional environment variables to set for the command. /// Allows a caller to provide interactive input and also handle /// both output and error streams from a single delegate. public static void RunAsync( string toolPath, string arguments, CompletionHandler completionDelegate, string workingDirectory = null, Dictionary envVars = null, IOHandler ioHandler = null) { Thread thread = new Thread(new ThreadStart(() => { Result result = Run(toolPath, arguments, workingDirectory, envVars: envVars, ioHandler: ioHandler); completionDelegate(result); })); thread.Start(); } /// /// Asynchronously reads binary data from a stream using a configurable buffer. /// private class AsyncStreamReader { /// /// Delegate called when data is read from the stream. /// Data read from the stream. /// public delegate void Handler(StreamData streamData); /// /// Event which is signalled when data is received. /// public event Handler DataReceived; // Signalled when a read completes. private AutoResetEvent readEvent = null; // Handle to the stream. private int handle; // Stream to read. private Stream stream; // Buffer used to read data from the stream. private byte[] buffer; // Whether reading is complete. volatile bool complete = false; /// /// Initialize the reader. /// /// Stream to read. /// Size of the buffer to read. public AsyncStreamReader(int handle, Stream stream, int bufferSize) { readEvent = new AutoResetEvent(false); this.handle = handle; this.stream = stream; buffer = new byte[bufferSize]; } /// /// Get the handle of the stream associated with this reader. /// public int Handle { get { return handle; } } /// /// Start reading. /// public void Start() { if (!complete) (new Thread(new ThreadStart(Read))).Start(); } /// /// Read from the stream until the end is reached. /// private void Read() { while (!complete) { stream.BeginRead( buffer, 0, buffer.Length, (asyncResult) => { int bytesRead = stream.EndRead(asyncResult); if (!complete) { complete = bytesRead == 0; if (DataReceived != null) { byte[] copy = new byte[bytesRead]; Array.Copy(buffer, copy, copy.Length); DataReceived(new StreamData( handle, System.Text.Encoding.UTF8.GetString(copy), copy, complete)); } } readEvent.Set(); }, null); readEvent.WaitOne(); } } /// /// Create a set of readers to read the specified streams, handles are assigned /// based upon the index of each stream in the provided array. /// /// Streams to read. /// Size of the buffer to use to read each stream. public static AsyncStreamReader[] CreateFromStreams(Stream[] streams, int bufferSize) { AsyncStreamReader[] readers = new AsyncStreamReader[streams.Length]; for (int i = 0; i < streams.Length; i++) { readers[i] = new AsyncStreamReader(i, streams[i], bufferSize); } return readers; } } /// /// Multiplexes data read from multiple AsyncStreamReaders onto a single thread. /// private class AsyncStreamReaderMultiplexer { /// Used to wait on items in the queue. private AutoResetEvent queuedItem = null; /// Queue of Data read from the readers. private System.Collections.Queue queue = null; /// Active stream handles. private HashSet activeStreams; /// /// Called when all streams reach the end or the reader is shut down. /// public delegate void CompletionHandler(); /// /// Called when all streams reach the end or the reader is shut down. /// public event CompletionHandler Complete; /// /// Handler called from the multiplexer's thread. /// public event AsyncStreamReader.Handler DataReceived; /// /// Create the multiplexer and attach it to the specified handler. /// /// Readers to read. /// Called for queued data item. /// Called when all readers complete. public AsyncStreamReaderMultiplexer(AsyncStreamReader[] readers, AsyncStreamReader.Handler handler, CompletionHandler complete = null) { queuedItem = new AutoResetEvent(false); queue = System.Collections.Queue.Synchronized(new System.Collections.Queue()); activeStreams = new HashSet(); foreach (AsyncStreamReader reader in readers) { reader.DataReceived += HandleRead; activeStreams.Add(reader.Handle); } DataReceived += handler; if (complete != null) Complete += complete; (new Thread(new ThreadStart(PollQueue))).Start(); } /// /// Shutdown the multiplexer. /// public void Shutdown() { lock (activeStreams) { activeStreams.Clear(); } queuedItem.Set(); } // Handle stream read notification. private void HandleRead(StreamData streamData) { queue.Enqueue(streamData); queuedItem.Set(); } // Poll the queue. private void PollQueue() { while (activeStreams.Count > 0) { queuedItem.WaitOne(); while (queue.Count > 0) { StreamData data = (StreamData)queue.Dequeue(); if (data.end) { lock(activeStreams) { activeStreams.Remove(data.handle); } } if (DataReceived != null) DataReceived(data); } } if (Complete != null) Complete(); } } /// /// Aggregates lines read by AsyncStreamReader. /// public class LineReader { // List of data keyed by stream handle. private Dictionary> streamDataByHandle = new Dictionary>(); /// /// Event called with a new line of data. /// public event IOHandler LineHandler; /// /// Event called for each piece of data received. /// public event IOHandler DataHandler; /// /// Initialize the instance. /// /// Called for each line read. public LineReader(IOHandler handler = null) { if (handler != null) LineHandler += handler; } /// /// Retrieve the currently buffered set of data. /// This allows the user to retrieve data before the end of a stream when /// a newline isn't present. /// /// Handle of the stream to query. /// List of data for the requested stream. public List GetBufferedData(int handle) { List handleData; return streamDataByHandle.TryGetValue(handle, out handleData) ? new List(handleData) : new List(); } /// /// Flush the internal buffer. /// public void Flush() { foreach (List handleData in streamDataByHandle.Values) { handleData.Clear(); } } /// /// Aggregate the specified list of StringBytes into a single structure. /// /// Stream handle. /// Data to aggregate. public static StreamData Aggregate(List dataStream) { // Build a list of strings up to the newline. List stringList = new List(); int byteArraySize = 0; int handle = 0; bool end = false; foreach (StreamData sd in dataStream) { stringList.Add(sd.text); byteArraySize += sd.data.Length; handle = sd.handle; end |= sd.end; } string concatenatedString = String.Join("", stringList.ToArray()); // Concatenate the list of bytes up to the StringBytes before the // newline. byte[] byteArray = new byte[byteArraySize]; int byteArrayOffset = 0; foreach (StreamData sd in dataStream) { Array.Copy(sd.data, 0, byteArray, byteArrayOffset, sd.data.Length); byteArrayOffset += sd.data.Length; } return new StreamData(handle, concatenatedString, byteArray, end); } /// /// Delegate method which can be attached to AsyncStreamReader.DataReceived to /// aggregate data until a new line is found before calling LineHandler. /// public void AggregateLine(Process process, StreamWriter stdin, StreamData data) { if (DataHandler != null) DataHandler(process, stdin, data); bool linesProcessed = false; if (data.data != null) { // Simplify line tracking by converting linefeeds to newlines. data.text = data.text.Replace("\r\n", "\n").Replace("\r", "\n"); List aggregateList = GetBufferedData(data.handle); aggregateList.Add(data); bool complete = false; while (!complete) { List newAggregateList = new List(); int textDataIndex = 0; int aggregateListSize = aggregateList.Count; complete = true; foreach (StreamData textData in aggregateList) { bool end = data.end && (++textDataIndex == aggregateListSize); newAggregateList.Add(textData); int newlineOffset = textData.text.Length; if (!end) { newlineOffset = textData.text.IndexOf("\n"); if (newlineOffset < 0) continue; newAggregateList.Remove(textData); } StreamData concatenated = Aggregate(newAggregateList); // Flush the aggregation list and store the overflow. newAggregateList.Clear(); if (!end) { // Add the remaining line to the concatenated data. concatenated.text += textData.text.Substring(0, newlineOffset + 1); // Save the line after the newline in the buffer. newAggregateList.Add(new StreamData( data.handle, textData.text.Substring(newlineOffset + 1), textData.data, false)); complete = false; } // Report the data. if (LineHandler != null) LineHandler(process, stdin, concatenated); linesProcessed = true; } aggregateList = newAggregateList; } streamDataByHandle[data.handle] = aggregateList; } // If no lines were processed call the handle to allow it to look ahead. if (!linesProcessed && LineHandler != null) { LineHandler(process, stdin, StreamData.Empty); } } } /// /// Execute a command line tool. /// /// Tool to execute. /// String to pass to the tools' command line. /// Directory to execute the tool from. /// Additional environment variables to set for the command. /// Allows a caller to provide interactive input and also handle /// both output and error streams from a single delegate. /// CommandLineTool result if successful, raises an exception if it's not /// possible to execute the tool. public static Result Run(string toolPath, string arguments, string workingDirectory = null, Dictionary envVars = null, IOHandler ioHandler = null) { return RunViaShell(toolPath, arguments, workingDirectory: workingDirectory, envVars: envVars, ioHandler: ioHandler, useShellExecution: false); } /// /// Execute a command line tool. /// /// Tool to execute. On Windows, if the path to this tool contains /// single quotes (apostrophes) the tool will be executed via the shell. /// String to pass to the tools' command line. /// Directory to execute the tool from. /// Additional environment variables to set for the command. /// Allows a caller to provide interactive input and also handle /// both output and error streams from a single delegate. NOTE: This is ignored if /// shell execution is enabled. /// Execute the command via the shell. This disables /// I/O redirection and causes a window to be popped up when the command is executed. /// This uses file redirection to retrieve the standard output stream. /// /// Enables stdout and stderr redirection when /// executing a command via the shell. This requires: /// * cmd.exe (on Windows) or bash (on OSX / Linux) are in the path. /// * Arguments containing whitespace are quoted. /// CommandLineTool result if successful, raises an exception if it's not /// possible to execute the tool. public static Result RunViaShell( string toolPath, string arguments, string workingDirectory = null, Dictionary envVars = null, IOHandler ioHandler = null, bool useShellExecution = false, bool stdoutRedirectionInShellMode = true) { var inputEncoding = Console.InputEncoding; var outputEncoding = Console.OutputEncoding; // Set encoderShouldEmitUTF8Identifier to false to prevent writing a Byte Order Marker (BOM). Console.InputEncoding = new UTF8Encoding(false); Console.OutputEncoding = new UTF8Encoding(false); try { return RunViaShellInternal(toolPath, arguments, workingDirectory, envVars, ioHandler, useShellExecution, stdoutRedirectionInShellMode); } finally { Console.InputEncoding = inputEncoding; Console.OutputEncoding = outputEncoding; } } private static Result RunViaShellInternal( string toolPath, string arguments, string workingDirectory, Dictionary envVars, IOHandler ioHandler, bool useShellExecution, bool stdoutRedirectionInShellMode) { // Mono 3.x on Windows can't execute tools with single quotes (apostrophes) in the path. // The following checks for this condition and forces shell execution of tools in these // paths which works fine as the shell tool should be in the system PATH. if (UnityEngine.RuntimePlatform.WindowsEditor == UnityEngine.Application.platform && toolPath.Contains("\'")) { useShellExecution = true; stdoutRedirectionInShellMode = true; } string stdoutFileName = null; string stderrFileName = null; if (useShellExecution && stdoutRedirectionInShellMode) { stdoutFileName = Path.GetTempFileName(); stderrFileName = Path.GetTempFileName(); string shellCmd ; string shellArgPrefix; string shellArgPostfix; string escapedToolPath = toolPath; if (UnityEngine.RuntimePlatform.WindowsEditor == UnityEngine.Application.platform) { shellCmd = "cmd.exe"; shellArgPrefix = "/c \""; shellArgPostfix = "\""; } else { shellCmd = "bash"; shellArgPrefix = "-l -c '"; shellArgPostfix = "'"; escapedToolPath = toolPath.Replace("'", "'\\''"); } arguments = String.Format("{0}\"{1}\" {2} 1> {3} 2> {4}{5}", shellArgPrefix, escapedToolPath, arguments, stdoutFileName, stderrFileName, shellArgPostfix); toolPath = shellCmd; } Process process = new Process(); process.StartInfo.UseShellExecute = useShellExecution; process.StartInfo.Arguments = arguments; if (useShellExecution) { process.StartInfo.CreateNoWindow = false; process.StartInfo.RedirectStandardOutput = false; process.StartInfo.RedirectStandardError = false; } else { process.StartInfo.CreateNoWindow = true; process.StartInfo.RedirectStandardOutput = true; process.StartInfo.RedirectStandardError = true; if (envVars != null) { foreach (var env in envVars) { process.StartInfo.EnvironmentVariables[env.Key] = env.Value; } } } process.StartInfo.RedirectStandardInput = !useShellExecution && (ioHandler != null); process.StartInfo.FileName = toolPath; process.StartInfo.WorkingDirectory = workingDirectory ?? Environment.CurrentDirectory; var started = process.Start(); if (!started) { UnityEngine.Debug.LogErrorFormat("Failed to start {0}", process); return new Result {exitCode = -1}; } // If an I/O handler was specified, call it with no data to provide a process and stdin // handle before output data is sent to it. if (ioHandler != null) ioHandler(process, process.StandardInput, StreamData.Empty); List[] stdouterr = new List[] { new List(), new List() }; if (useShellExecution) { process.WaitForExit(); if (stdoutRedirectionInShellMode) { stdouterr[0].Add(File.ReadAllText(stdoutFileName)); stdouterr[1].Add(File.ReadAllText(stderrFileName)); File.Delete(stdoutFileName); File.Delete(stderrFileName); } } else { AutoResetEvent complete = new AutoResetEvent(false); // Read raw output from the process. AsyncStreamReader[] readers = AsyncStreamReader.CreateFromStreams( new Stream[] { process.StandardOutput.BaseStream, process.StandardError.BaseStream }, 1); new AsyncStreamReaderMultiplexer( readers, (StreamData data) => { stdouterr[data.handle].Add(data.text); if (ioHandler != null) ioHandler(process, process.StandardInput, data); }, complete: () => { complete.Set(); }); foreach (AsyncStreamReader reader in readers) reader.Start(); process.WaitForExit(); // Wait for the reading threads to complete. complete.WaitOne(); } Result result = new Result(); result.stdout = String.Join("", stdouterr[StandardOutputStreamDataHandle].ToArray()); result.stderr = String.Join("", stdouterr[StandardErrorStreamDataHandle].ToArray()); result.exitCode = process.ExitCode; result.message = FormatResultMessage(toolPath, arguments, result.stdout, result.stderr, result.exitCode); return result; } /// /// Split a string into lines using newline, carriage return or a combination of both. /// /// String to split into lines public static string[] SplitLines(string multilineString) { return Regex.Split(multilineString, "\r\n|\r|\n"); } /// /// Format a command excecution error message. /// /// Tool executed. /// Arguments used to execute the tool. /// Standard output stream from tool execution. /// Standard error stream from tool execution. /// Result of the tool. private static string FormatResultMessage(string toolPath, string arguments, string stdout, string stderr, int exitCode) { return String.Format( "{0} '{1} {2}'\n" + "stdout:\n" + "{3}\n" + "stderr:\n" + "{4}\n" + "exit code: {5}\n", exitCode == 0 ? "Successfully executed" : "Failed to run", toolPath, arguments, stdout, stderr, exitCode); } /// /// Returns the specified argument in double quotes. /// public static string QuotePath(string path) { return string.Format("\"{0}\"", path); } #if UNITY_EDITOR /// /// Get an executable extension. /// /// Platform specific extension for executables. public static string GetExecutableExtension() { return (UnityEngine.RuntimePlatform.WindowsEditor == UnityEngine.Application.platform) ? ".exe" : ""; } /// /// Locate an executable in the system path. /// /// Executable name without a platform specific extension like /// .exe /// A string to the executable path if it's found, null otherwise. public static string FindExecutable(string executable) { string which = (UnityEngine.RuntimePlatform.WindowsEditor == UnityEngine.Application.platform) ? "where" : "which"; try { Result result = Run(which, executable, Environment.CurrentDirectory); if (result.exitCode == 0) { return SplitLines(result.stdout)[0]; } } catch (Exception e) { UnityEngine.Debug.Log("'" + which + "' command is not on path. " + "Unable to find executable '" + executable + "' (" + e.ToString() + ")"); } return null; } #endif // UNITY_EDITOR } }