View Javadoc
1   /*
2    * Copyright (c) 2002-2017 Gargoyle Software Inc.
3    *
4    * Licensed under the Apache License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    * http://www.apache.org/licenses/LICENSE-2.0
8    *
9    * Unless required by applicable law or agreed to in writing, software
10   * distributed under the License is distributed on an "AS IS" BASIS,
11   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12   * See the License for the specific language governing permissions and
13   * limitations under the License.
14   */
15  package com.gargoylesoftware.htmlunit.util;
16  
17  import static java.nio.charset.StandardCharsets.ISO_8859_1;
18  
19  import java.io.EOFException;
20  import java.io.File;
21  import java.io.FileOutputStream;
22  import java.io.FileWriter;
23  import java.io.IOException;
24  import java.io.InputStream;
25  import java.io.OutputStream;
26  import java.net.URL;
27  import java.util.ArrayList;
28  import java.util.List;
29  import java.util.regex.Pattern;
30  
31  import org.apache.commons.io.FileUtils;
32  import org.apache.commons.io.IOUtils;
33  import org.apache.commons.lang3.StringUtils;
34  import org.apache.commons.logging.Log;
35  import org.apache.commons.logging.LogFactory;
36  
37  import com.gargoylesoftware.htmlunit.FormEncodingType;
38  import com.gargoylesoftware.htmlunit.HttpMethod;
39  import com.gargoylesoftware.htmlunit.WebConnection;
40  import com.gargoylesoftware.htmlunit.WebRequest;
41  import com.gargoylesoftware.htmlunit.WebResponse;
42  import com.gargoylesoftware.htmlunit.WebResponseData;
43  
44  import net.sourceforge.htmlunit.corejs.javascript.Context;
45  import net.sourceforge.htmlunit.corejs.javascript.ContextAction;
46  import net.sourceforge.htmlunit.corejs.javascript.ContextFactory;
47  import net.sourceforge.htmlunit.corejs.javascript.Script;
48  
49  /**
50   * Wrapper around a "real" WebConnection that will use the wrapped web connection
51   * to do the real job and save all received responses
52   * in the temp directory with an overview page.<br>
53   * <br>
54   * This may be useful at conception time to understand what is "browsed".<br>
55   * <br>
56   * Example:
57   * <pre>
58   * final WebClient client = new WebClient();
59   * final WebConnection connection = new DebuggingWebConnection(client.getWebConnection(), "myTest");
60   * client.setWebConnection(connection);
61   * </pre>
62   * In this example an overview page will be generated under the name myTest/index.html in the temp directory
63   * and all received responses will be saved into the myTest folder.<br>
64   * <br>
65   * <em>This class is only intended as an help during the conception.</em>
66   *
67   * @author Marc Guillemot
68   * @author Ahmed Ashour
69   * @author Ronald Brill
70   */
71  public class DebuggingWebConnection extends WebConnectionWrapper {
72      private static final Log LOG = LogFactory.getLog(DebuggingWebConnection.class);
73  
74      private static final Pattern ESCAPE_QUOTE_PATTERN = Pattern.compile("'");
75  
76      private int counter_;
77      private final WebConnection wrappedWebConnection_;
78      private final File javaScriptFile_;
79      private final File reportFolder_;
80      private boolean uncompressJavaScript_ = true;
81  
82      /**
83       * Wraps a web connection to have a report generated of the received responses.
84       * @param webConnection the webConnection that do the real work
85       * @param dirName the name of the directory to create in the tmp folder to save received responses.
86       * If this folder already exists, it will be deleted first.
87       * @throws IOException in case of problems writing the files
88       */
89      public DebuggingWebConnection(final WebConnection webConnection,
90              final String dirName) throws IOException {
91  
92          super(webConnection);
93  
94          wrappedWebConnection_ = webConnection;
95          final File tmpDir = new File(System.getProperty("java.io.tmpdir"));
96          reportFolder_ = new File(tmpDir, dirName);
97          if (reportFolder_.exists()) {
98              FileUtils.forceDelete(reportFolder_);
99          }
100         FileUtils.forceMkdir(reportFolder_);
101         javaScriptFile_ = new File(reportFolder_, "hu.js");
102         createOverview();
103     }
104 
105     /**
106      * Calls the wrapped webconnection and save the received response.
107      * {@inheritDoc}
108      */
109     @Override
110     public WebResponse getResponse(final WebRequest request) throws IOException {
111         WebResponse response = wrappedWebConnection_.getResponse(request);
112         if (isUncompressJavaScript() && isJavaScript(response.getContentType())) {
113             response = uncompressJavaScript(response);
114         }
115         saveResponse(response, request);
116         return response;
117     }
118 
119     /**
120      * Tries to uncompress the JavaScript code in the provided response.
121      * @param response the response to uncompress
122      * @return a new response with uncompressed JavaScript code or the original response in case of failure
123      */
124     protected WebResponse uncompressJavaScript(final WebResponse response) {
125         final WebRequest request = response.getWebRequest();
126         final String scriptName = request.getUrl().toString();
127         final String scriptSource = response.getContentAsString();
128 
129         // skip if it is already formatted? => TODO
130 
131         final ContextFactory factory = new ContextFactory();
132         final ContextAction action = new ContextAction() {
133             @Override
134             public Object run(final Context cx) {
135                 cx.setOptimizationLevel(-1);
136                 final Script script = cx.compileString(scriptSource, scriptName, 0, null);
137                 return cx.decompileScript(script, 4);
138             }
139         };
140 
141         try {
142             final String decompileScript = (String) factory.call(action);
143             final List<NameValuePair> responseHeaders = new ArrayList<>(response.getResponseHeaders());
144             for (int i = responseHeaders.size() - 1; i >= 0; i--) {
145                 if ("content-encoding".equalsIgnoreCase(responseHeaders.get(i).getName())) {
146                     responseHeaders.remove(i);
147                 }
148             }
149             final WebResponseData wrd = new WebResponseData(decompileScript.getBytes(), response.getStatusCode(),
150                 response.getStatusMessage(), responseHeaders);
151             return new WebResponse(wrd, response.getWebRequest().getUrl(),
152                 response.getWebRequest().getHttpMethod(), response.getLoadTime());
153         }
154         catch (final Exception e) {
155             LOG.warn("Failed to decompress JavaScript response. Delivering as it.", e);
156         }
157 
158         return response;
159     }
160 
161     /**
162      * Adds a mark that will be visible in the HTML result page generated by this class.
163      * @param mark the text
164      * @throws IOException if a problem occurs writing the file
165      */
166     public void addMark(String mark) throws IOException {
167         if (mark != null) {
168             mark = mark.replace("\"", "\\\"");
169         }
170         appendToJSFile("tab[tab.length] = \"" + mark + "\";\n");
171         LOG.info("--- " + mark + " ---");
172     }
173 
174     /**
175      * Saves the response content in the temp dir and adds it to the summary page.
176      * @param response the response to save
177      * @param request the request used to get the response
178      * @throws IOException if a problem occurs writing the file
179      */
180     protected void saveResponse(final WebResponse response, final WebRequest request)
181         throws IOException {
182         counter_++;
183         final String extension = chooseExtension(response.getContentType());
184         final File f = createFile(request.getUrl(), extension);
185         int length = 0;
186         try (InputStream input = response.getContentAsStream()) {
187             try (OutputStream output = new FileOutputStream(f)) {
188                 length = IOUtils.copy(input, output);
189             }
190             catch (final EOFException e) {
191                 // ignore
192             }
193         }
194 
195         final URL url = response.getWebRequest().getUrl();
196         LOG.info("Created file " + f.getAbsolutePath() + " for response " + counter_ + ": " + url);
197 
198         final StringBuilder bduiler = new StringBuilder();
199         bduiler.append("tab[tab.length] = {code: " + response.getStatusCode() + ", ");
200         bduiler.append("fileName: '" + f.getName() + "', ");
201         bduiler.append("contentType: '" + response.getContentType() + "', ");
202         bduiler.append("method: '" + request.getHttpMethod().name() + "', ");
203         if (request.getHttpMethod() == HttpMethod.POST && request.getEncodingType() == FormEncodingType.URL_ENCODED) {
204             bduiler.append("postParameters: " + nameValueListToJsMap(request.getRequestParameters()) + ", ");
205         }
206         bduiler.append("url: '" + escapeJSString(url.toString()) + "', ");
207         bduiler.append("loadTime: " + response.getLoadTime() + ", ");
208         bduiler.append("responseSize: " + length + ", ");
209         bduiler.append("responseHeaders: " + nameValueListToJsMap(response.getResponseHeaders()));
210         bduiler.append("};\n");
211         appendToJSFile(bduiler.toString());
212     }
213 
214     static String escapeJSString(final String string) {
215         return ESCAPE_QUOTE_PATTERN.matcher(string).replaceAll("\\\\'");
216     }
217 
218     static String chooseExtension(final String contentType) {
219         if (isJavaScript(contentType)) {
220             return ".js";
221         }
222         else if ("text/html".equals(contentType)) {
223             return ".html";
224         }
225         else if ("text/css".equals(contentType)) {
226             return ".css";
227         }
228         else if ("text/xml".equals(contentType)) {
229             return ".xml";
230         }
231         else if ("image/gif".equals(contentType)) {
232             return ".gif";
233         }
234         return ".txt";
235     }
236 
237     /**
238      * Indicates if the response contains JavaScript content.
239      * @param contentType the response's content type
240      * @return {@code false} if it is not recognized as JavaScript
241      */
242     static boolean isJavaScript(final String contentType) {
243         return contentType.contains("javascript") || contentType.contains("ecmascript")
244             || (contentType.startsWith("text/") && contentType.endsWith("js"));
245     }
246 
247     /**
248      * Indicates if it should try to format responses recognized as JavaScript.
249      * @return default is {@code false} to deliver the original content
250      */
251     public boolean isUncompressJavaScript() {
252         return uncompressJavaScript_;
253     }
254 
255     /**
256      * Indicates that responses recognized as JavaScript should be formatted or not.
257      * Formatting is interesting for debugging when the original script is compressed on a single line.
258      * It allows to better follow with a debugger and to obtain more interesting error messages.
259      * @param decompress {@code true} if JavaScript responses should be uncompressed
260      */
261     public void setUncompressJavaScript(final boolean decompress) {
262         uncompressJavaScript_ = decompress;
263     }
264 
265     private void appendToJSFile(final String str) throws IOException {
266         try (FileWriter jsFileWriter = new FileWriter(javaScriptFile_, true)) {
267             jsFileWriter.write(str);
268         }
269     }
270 
271     /**
272      * Computes the best file to save the response to the given URL.
273      * @param url the requested URL
274      * @param extension the preferred extension
275      * @return the file to create
276      * @throws IOException if a problem occurs creating the file
277      */
278     private File createFile(final URL url, final String extension) throws IOException {
279         String name = url.getPath().replaceFirst("/$", "").replaceAll(".*/", "");
280         name = StringUtils.substringBefore(name, "?"); // remove query
281         name = StringUtils.substringBefore(name, ";"); // remove additional info
282         name = StringUtils.substring(name, 0, 30); // avoid exceptions due to too long file names
283         name = com.gargoylesoftware.htmlunit.util.StringUtils.sanitizeForFileName(name);
284         if (!name.endsWith(extension)) {
285             name += extension;
286         }
287         int counter = 0;
288         while (true) {
289             final String fileName;
290             if (counter != 0) {
291                 fileName = StringUtils.substringBeforeLast(name, ".")
292                     + "_" + counter + "." + StringUtils.substringAfterLast(name, ".");
293             }
294             else {
295                 fileName = name;
296             }
297             final File f = new File(reportFolder_, fileName);
298             if (f.createNewFile()) {
299                 return f;
300             }
301             counter++;
302         }
303     }
304 
305     /**
306      * Produces a String that will produce a JS map like "{'key1': 'value1', 'key 2': 'value2'}".
307      * @param headers a list of {@link NameValuePair}
308      * @return the JS String
309      */
310     static String nameValueListToJsMap(final List<NameValuePair> headers) {
311         if (headers == null || headers.isEmpty()) {
312             return "{}";
313         }
314         final StringBuilder bduiler = new StringBuilder("{");
315         for (final NameValuePair header : headers) {
316             bduiler.append("'" + header.getName() + "': '" + escapeJSString(header.getValue()) + "', ");
317         }
318         bduiler.delete(bduiler.length() - 2, bduiler.length());
319         bduiler.append("}");
320         return bduiler.toString();
321     }
322 
323     /**
324      * Creates the summary file and the JavaScript file that will be updated for each received response
325      * @throws IOException if a problem occurs writing the file
326      */
327     private void createOverview() throws IOException {
328         FileUtils.writeStringToFile(javaScriptFile_, "var tab = [];\n", ISO_8859_1);
329 
330         final URL indexResource = DebuggingWebConnection.class.getResource("DebuggingWebConnection.index.html");
331         if (indexResource == null) {
332             throw new RuntimeException("Missing dependency DebuggingWebConnection.index.html");
333         }
334         final File summary = new File(reportFolder_, "index.html");
335         FileUtils.copyURLToFile(indexResource, summary);
336 
337         LOG.info("Summary will be in " + summary.getAbsolutePath());
338     }
339 
340     File getReportFolder() {
341         return reportFolder_;
342     }
343 }