View Javadoc
1   /*
2    * Copyright (c) 2002-2017 Gargoyle Software Inc.
3    *
4    * Licensed under the Apache License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    * http://www.apache.org/licenses/LICENSE-2.0
8    *
9    * Unless required by applicable law or agreed to in writing, software
10   * distributed under the License is distributed on an "AS IS" BASIS,
11   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12   * See the License for the specific language governing permissions and
13   * limitations under the License.
14   */
15  package com.gargoylesoftware.htmlunit;
16  
17  import static com.gargoylesoftware.htmlunit.BrowserVersionFeatures.DIALOGWINDOW_REFERER;
18  import static com.gargoylesoftware.htmlunit.BrowserVersionFeatures.HTTP_REDIRECT_308;
19  import static com.gargoylesoftware.htmlunit.BrowserVersionFeatures.JS_XML_SUPPORT_VIA_ACTIVEXOBJECT;
20  import static com.gargoylesoftware.htmlunit.BrowserVersionFeatures.URL_MINIMAL_QUERY_ENCODING;
21  import static com.gargoylesoftware.htmlunit.BrowserVersionFeatures.WINDOW_EXECUTE_EVENTS;
22  import static java.nio.charset.StandardCharsets.UTF_8;
23  
24  import java.io.BufferedInputStream;
25  import java.io.File;
26  import java.io.FileInputStream;
27  import java.io.IOException;
28  import java.io.InputStream;
29  import java.io.ObjectInputStream;
30  import java.io.Serializable;
31  import java.lang.ref.WeakReference;
32  import java.net.MalformedURLException;
33  import java.net.URL;
34  import java.net.URLConnection;
35  import java.net.URLDecoder;
36  import java.nio.charset.Charset;
37  import java.util.ArrayList;
38  import java.util.Collections;
39  import java.util.ConcurrentModificationException;
40  import java.util.Date;
41  import java.util.HashMap;
42  import java.util.HashSet;
43  import java.util.Iterator;
44  import java.util.LinkedHashSet;
45  import java.util.List;
46  import java.util.Locale;
47  import java.util.Map;
48  import java.util.Set;
49  import java.util.Stack;
50  
51  import org.apache.commons.codec.DecoderException;
52  import org.apache.commons.lang3.StringUtils;
53  import org.apache.commons.logging.Log;
54  import org.apache.commons.logging.LogFactory;
55  import org.apache.http.HttpStatus;
56  import org.apache.http.NoHttpResponseException;
57  import org.apache.http.client.CredentialsProvider;
58  import org.apache.http.cookie.ClientCookie;
59  import org.apache.http.cookie.CookieOrigin;
60  import org.apache.http.cookie.CookieSpec;
61  import org.apache.http.cookie.MalformedCookieException;
62  import org.apache.http.message.BufferedHeader;
63  import org.apache.http.util.CharArrayBuffer;
64  import org.w3c.css.sac.ErrorHandler;
65  
66  import com.gargoylesoftware.htmlunit.activex.javascript.msxml.MSXMLActiveXObjectFactory;
67  import com.gargoylesoftware.htmlunit.attachment.Attachment;
68  import com.gargoylesoftware.htmlunit.attachment.AttachmentHandler;
69  import com.gargoylesoftware.htmlunit.gae.GAEUtils;
70  import com.gargoylesoftware.htmlunit.html.BaseFrameElement;
71  import com.gargoylesoftware.htmlunit.html.DomElement;
72  import com.gargoylesoftware.htmlunit.html.DomNode;
73  import com.gargoylesoftware.htmlunit.html.FrameWindow;
74  import com.gargoylesoftware.htmlunit.html.HTMLParserListener;
75  import com.gargoylesoftware.htmlunit.html.HtmlInlineFrame;
76  import com.gargoylesoftware.htmlunit.html.HtmlPage;
77  import com.gargoylesoftware.htmlunit.httpclient.HtmlUnitBrowserCompatCookieSpec;
78  import com.gargoylesoftware.htmlunit.javascript.AbstractJavaScriptEngine;
79  import com.gargoylesoftware.htmlunit.javascript.DefaultJavaScriptErrorListener;
80  import com.gargoylesoftware.htmlunit.javascript.JavaScriptEngine;
81  import com.gargoylesoftware.htmlunit.javascript.JavaScriptErrorListener;
82  import com.gargoylesoftware.htmlunit.javascript.background.JavaScriptJobManager;
83  import com.gargoylesoftware.htmlunit.javascript.host.Location;
84  import com.gargoylesoftware.htmlunit.javascript.host.Window;
85  import com.gargoylesoftware.htmlunit.javascript.host.css.ComputedCSSStyleDeclaration;
86  import com.gargoylesoftware.htmlunit.javascript.host.dom.Node;
87  import com.gargoylesoftware.htmlunit.javascript.host.event.Event;
88  import com.gargoylesoftware.htmlunit.javascript.host.html.HTMLDocument;
89  import com.gargoylesoftware.htmlunit.javascript.host.html.HTMLElement;
90  import com.gargoylesoftware.htmlunit.javascript.host.html.HTMLIFrameElement;
91  import com.gargoylesoftware.htmlunit.protocol.data.DataUrlDecoder;
92  import com.gargoylesoftware.htmlunit.util.Cookie;
93  import com.gargoylesoftware.htmlunit.util.NameValuePair;
94  import com.gargoylesoftware.htmlunit.util.UrlUtils;
95  
96  import net.sourceforge.htmlunit.corejs.javascript.ScriptableObject;
97  
98  /**
99   * The main starting point in HtmlUnit: this class simulates a web browser.
100  * <p>
101  * A standard usage of HtmlUnit will start with using the {@link #getPage(String)} method
102  * (or {@link #getPage(URL)}) to load a first {@link Page}
103  * and will continue with further processing on this page depending on its type.
104  * </p>
105  * <b>Example:</b><br>
106  * <br>
107  * <code>
108  * final WebClient webClient = new WebClient();<br>
109  * final {@link HtmlPage} startPage = webClient.getPage("http://htmlunit.sf.net");<br>
110  * assertEquals("HtmlUnit - Welcome to HtmlUnit", startPage.{@link HtmlPage#getTitleText() getTitleText}());
111  * </code>
112  * <p>
113  * Note: a {@link WebClient} instance is <b>not thread safe</b>. It is intended to be used from a single thread.
114  * </p>
115  * @author <a href="mailto:mbowler@GargoyleSoftware.com">Mike Bowler</a>
116  * @author <a href="mailto:gudujarlson@sf.net">Mike J. Bresnahan</a>
117  * @author Dominique Broeglin
118  * @author Noboru Sinohara
119  * @author <a href="mailto:chen_jun@users.sourceforge.net">Chen Jun</a>
120  * @author David K. Taylor
121  * @author <a href="mailto:cse@dynabean.de">Christian Sell</a>
122  * @author <a href="mailto:bcurren@esomnie.com">Ben Curren</a>
123  * @author Marc Guillemot
124  * @author Chris Erskine
125  * @author Daniel Gredler
126  * @author Sergey Gorelkin
127  * @author Hans Donner
128  * @author Paul King
129  * @author Ahmed Ashour
130  * @author Bruce Chapman
131  * @author Sudhan Moghe
132  * @author Martin Tamme
133  * @author Amit Manjhi
134  * @author Nicolas Belisle
135  * @author Ronald Brill
136  * @author Frank Danek
137  * @author Joerg Werner
138  */
139 public class WebClient implements Serializable, AutoCloseable {
140 
141     /** Logging support. */
142     private static final Log LOG = LogFactory.getLog(WebClient.class);
143 
144     /** Like the Firefox default value for {@code network.http.redirection-limit}. */
145     private static final int ALLOWED_REDIRECTIONS_SAME_URL = 20;
146 
147     private transient WebConnection webConnection_;
148     private CredentialsProvider credentialsProvider_ = new DefaultCredentialsProvider();
149     private CookieManager cookieManager_ = new CookieManager();
150     private transient AbstractJavaScriptEngine<?> scriptEngine_;
151     private final Map<String, String> requestHeaders_ = Collections.synchronizedMap(new HashMap<String, String>(89));
152     private IncorrectnessListener incorrectnessListener_ = new IncorrectnessListenerImpl();
153     private WebConsole webConsole_;
154 
155     private AlertHandler alertHandler_;
156     private ConfirmHandler confirmHandler_;
157     private PromptHandler promptHandler_;
158     private StatusHandler statusHandler_;
159     private AttachmentHandler attachmentHandler_;
160     private AppletConfirmHandler appletConfirmHandler_;
161     private AjaxController ajaxController_ = new AjaxController();
162 
163     private BrowserVersion browserVersion_;
164     private PageCreator pageCreator_ = new DefaultPageCreator();
165 
166     private final Set<WebWindowListener> webWindowListeners_ = new HashSet<>(5);
167     private final Stack<TopLevelWindow> topLevelWindows_ = new Stack<>(); // top-level windows
168     private final List<WebWindow> windows_ = Collections.synchronizedList(new ArrayList<WebWindow>()); // all windows
169     private transient List<WeakReference<JavaScriptJobManager>> jobManagers_ =
170             Collections.synchronizedList(new ArrayList<WeakReference<JavaScriptJobManager>>());
171     private WebWindow currentWindow_;
172 
173     private HTMLParserListener htmlParserListener_;
174     private ErrorHandler cssErrorHandler_ = new DefaultCssErrorHandler();
175     private OnbeforeunloadHandler onbeforeunloadHandler_;
176     private Cache cache_ = new Cache();
177 
178     /** target "_blank". */
179     private static final String TARGET_BLANK = "_blank";
180     /** target "_parent". */
181     private static final String TARGET_SELF = "_self";
182     /** target "_parent". */
183     private static final String TARGET_PARENT = "_parent";
184     /** target "_top". */
185     private static final String TARGET_TOP = "_top";
186 
187     /** "about:". */
188     public static final String ABOUT_SCHEME = "about:";
189     /** "about:blank". */
190     public static final String ABOUT_BLANK = ABOUT_SCHEME + "blank";
191     /** URL for "about:blank". */
192     public static final URL URL_ABOUT_BLANK = UrlUtils.toUrlSafe(ABOUT_BLANK);
193 
194     private ScriptPreProcessor scriptPreProcessor_;
195 
196     private Map<String, String> activeXObjectMap_ = Collections.emptyMap();
197     private transient MSXMLActiveXObjectFactory msxmlActiveXObjectFactory_;
198     private RefreshHandler refreshHandler_ = new NiceRefreshHandler(2);
199     private JavaScriptErrorListener javaScriptErrorListener_ = new DefaultJavaScriptErrorListener();
200 
201     private WebClientOptions options_ = new WebClientOptions();
202     private WebClientInternals internals_ = new WebClientInternals(this);
203     private final StorageHolder storageHolder_ = new StorageHolder();
204 
205     private static final WebResponseData responseDataNoHttpResponse_ = new WebResponseData(
206         0, "No HTTP Response", Collections.<NameValuePair>emptyList());
207 
208     /**
209      * Creates a web client instance using the browser version returned by
210      * {@link BrowserVersion#getDefault()}.
211      */
212     public WebClient() {
213         this(BrowserVersion.getDefault());
214     }
215 
216     /**
217      * Creates a web client instance using the specified {@link BrowserVersion}.
218      * @param browserVersion the browser version to simulate
219      */
220     public WebClient(final BrowserVersion browserVersion) {
221         WebAssert.notNull("browserVersion", browserVersion);
222         init(browserVersion, new ProxyConfig());
223     }
224 
225     /**
226      * Creates an instance that will use the specified {@link BrowserVersion} and proxy server.
227      * @param browserVersion the browser version to simulate
228      * @param proxyHost the server that will act as proxy
229      * @param proxyPort the port to use on the proxy server
230      */
231     public WebClient(final BrowserVersion browserVersion, final String proxyHost, final int proxyPort) {
232         WebAssert.notNull("browserVersion", browserVersion);
233         WebAssert.notNull("proxyHost", proxyHost);
234         init(browserVersion, new ProxyConfig(proxyHost, proxyPort));
235     }
236 
237     /**
238      * Generic initialization logic used by all constructors. This method does not perform any
239      * parameter validation; such validation must be handled by the constructors themselves.
240      * @param browserVersion the browser version to simulate
241      * @param proxyConfig the proxy configuration to use
242      */
243     private void init(final BrowserVersion browserVersion, final ProxyConfig proxyConfig) {
244         browserVersion_ = browserVersion;
245         getOptions().setProxyConfig(proxyConfig);
246 
247         webConnection_ = createWebConnection(); // this has to be done after the browser version was set
248         scriptEngine_ = new JavaScriptEngine(this);
249         // The window must be constructed AFTER the script engine.
250         addWebWindowListener(new CurrentWindowTracker(this));
251         currentWindow_ = new TopLevelWindow("", this);
252         fireWindowOpened(new WebWindowEvent(currentWindow_, WebWindowEvent.OPEN, null, null));
253 
254         if (getBrowserVersion().hasFeature(JS_XML_SUPPORT_VIA_ACTIVEXOBJECT)) {
255             initMSXMLActiveX();
256         }
257     }
258 
259     private void initMSXMLActiveX() {
260         msxmlActiveXObjectFactory_ = new MSXMLActiveXObjectFactory();
261         // TODO [IE] initialize in #init or in #initialize?
262         try {
263             msxmlActiveXObjectFactory_.init(getBrowserVersion());
264         }
265         catch (final Exception e) {
266             LOG.error("Exception while initializing MSXML ActiveX for the page", e);
267             throw new ScriptException(null, e); // BUG: null is not useful.
268         }
269     }
270 
271     /**
272      * Returns the object that will resolve all URL requests.
273      *
274      * @return the connection that will be used
275      */
276     public WebConnection getWebConnection() {
277         return webConnection_;
278     }
279 
280     /**
281      * Sets the object that will resolve all URL requests.
282      *
283      * @param webConnection the new web connection
284      */
285     public void setWebConnection(final WebConnection webConnection) {
286         WebAssert.notNull("webConnection", webConnection);
287         webConnection_ = webConnection;
288     }
289 
290     /**
291      * Send a request to a server and return a Page that represents the
292      * response from the server. This page will be used to populate the provided window.
293      * <p>
294      * The returned {@link Page} will be created by the {@link PageCreator}
295      * configured by {@link #setPageCreator(PageCreator)}, if any.
296      * <p>
297      * The {@link DefaultPageCreator} will create a {@link Page} depending on the content type of the HTTP response,
298      * basically {@link HtmlPage} for HTML content, {@link com.gargoylesoftware.htmlunit.xml.XmlPage} for XML content,
299      * {@link TextPage} for other text content and {@link UnexpectedPage} for anything else.
300      *
301      * @param webWindow the WebWindow to load the result of the request into
302      * @param webRequest the web request
303      * @param <P> the page type
304      * @return the page returned by the server when the specified request was made in the specified window
305      * @throws IOException if an IO error occurs
306      * @throws FailingHttpStatusCodeException if the server returns a failing status code AND the property
307      *         {@link WebClientOptions#setThrowExceptionOnFailingStatusCode(boolean)} is set to true
308      *
309      * @see WebRequest
310      */
311     public <P extends Page> P getPage(final WebWindow webWindow, final WebRequest webRequest)
312             throws IOException, FailingHttpStatusCodeException {
313         return getPage(webWindow, webRequest, true);
314     }
315 
316     /**
317      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
318      *
319      * Send a request to a server and return a Page that represents the
320      * response from the server. This page will be used to populate the provided window.
321      * <p>
322      * The returned {@link Page} will be created by the {@link PageCreator}
323      * configured by {@link #setPageCreator(PageCreator)}, if any.
324      * <p>
325      * The {@link DefaultPageCreator} will create a {@link Page} depending on the content type of the HTTP response,
326      * basically {@link HtmlPage} for HTML content, {@link com.gargoylesoftware.htmlunit.xml.XmlPage} for XML content,
327      * {@link TextPage} for other text content and {@link UnexpectedPage} for anything else.
328      *
329      * @param webWindow the WebWindow to load the result of the request into
330      * @param webRequest the web request
331      * @param addToHistory true if the page should be part of the history
332      * @param <P> the page type
333      * @return the page returned by the server when the specified request was made in the specified window
334      * @throws IOException if an IO error occurs
335      * @throws FailingHttpStatusCodeException if the server returns a failing status code AND the property
336      *         {@link WebClientOptions#setThrowExceptionOnFailingStatusCode(boolean)} is set to true
337      *
338      * @see WebRequest
339      */
340     @SuppressWarnings("unchecked")
341     <P extends Page> P getPage(final WebWindow webWindow, final WebRequest webRequest,
342             final boolean addToHistory)
343         throws IOException, FailingHttpStatusCodeException {
344 
345         final Page page = webWindow.getEnclosedPage();
346 
347         if (page != null) {
348             final URL prev = page.getUrl();
349             final URL current = webRequest.getUrl();
350             if (UrlUtils.sameFile(current, prev)
351                         && current.getRef() != null
352                         && !StringUtils.equals(current.getRef(), prev.getRef())) {
353                 // We're just navigating to an anchor within the current page.
354                 page.getWebResponse().getWebRequest().setUrl(current);
355                 if (addToHistory) {
356                     webWindow.getHistory().addPage(page);
357                 }
358 
359                 final Window window = (Window) webWindow.getScriptableObject();
360                 if (window != null) { // js enabled
361                     window.getLocation().setHash(current.getRef());
362                     window.clearComputedStyles();
363                 }
364                 return (P) page;
365             }
366 
367             if (page.isHtmlPage()) {
368                 final HtmlPage htmlPage = (HtmlPage) page;
369                 if (!htmlPage.isOnbeforeunloadAccepted()) {
370                     if (LOG.isDebugEnabled()) {
371                         LOG.debug("The registered OnbeforeunloadHandler rejected to load a new page.");
372                     }
373                     return (P) page;
374                 }
375             }
376         }
377 
378         if (LOG.isDebugEnabled()) {
379             LOG.debug("Get page for window named '" + webWindow.getName() + "', using " + webRequest);
380         }
381 
382         final WebResponse webResponse;
383         final String protocol = webRequest.getUrl().getProtocol();
384         if ("javascript".equals(protocol)) {
385             webResponse = makeWebResponseForJavaScriptUrl(webWindow, webRequest.getUrl(), webRequest.getCharset());
386             if (webWindow.getEnclosedPage() != null && webWindow.getEnclosedPage().getWebResponse() == webResponse) {
387                 // a javascript:... url with result of type undefined didn't changed the page
388                 return (P) webWindow.getEnclosedPage();
389             }
390         }
391         else {
392             webResponse = loadWebResponse(webRequest);
393         }
394 
395         printContentIfNecessary(webResponse);
396         loadWebResponseInto(webResponse, webWindow);
397 
398         // start execution here
399         // note: we have to do this also if the server reports an error!
400         //       e.g. if the server returns a 404 error page that includes javascript
401         if (scriptEngine_ != null) {
402             scriptEngine_.registerWindowAndMaybeStartEventLoop(webWindow);
403         }
404 
405         // check and report problems if needed
406         throwFailingHttpStatusCodeExceptionIfNecessary(webResponse);
407         return (P) webWindow.getEnclosedPage();
408     }
409 
410     /**
411      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
412      *
413      * <p>Open a new web window and populate it with a page loaded by
414      * {@link #getPage(WebWindow,WebRequest)}</p>
415      *
416      * @param opener the web window that initiated the request
417      * @param target the name of the window to be opened (the name that will be passed into the
418      *        JavaScript <tt>open()</tt> method)
419      * @param params any parameters
420      * @param <P> the page type
421      * @return the new page
422      * @throws FailingHttpStatusCodeException if the server returns a failing status code AND the property
423      *         {@link WebClientOptions#setThrowExceptionOnFailingStatusCode(boolean)} is set to true.
424      * @throws IOException if an IO problem occurs
425      */
426     @SuppressWarnings("unchecked")
427     public <P extends Page> P getPage(final WebWindow opener, final String target, final WebRequest params)
428         throws FailingHttpStatusCodeException, IOException {
429         return (P) getPage(openTargetWindow(opener, target, TARGET_SELF), params);
430     }
431 
432     /**
433      * Convenient method to build an URL and load it into the current WebWindow as it would be done
434      * by {@link #getPage(WebWindow, WebRequest)}.
435      * @param url the URL of the new content
436      * @param <P> the page type
437      * @return the new page
438      * @throws FailingHttpStatusCodeException if the server returns a failing status code AND the property
439      *         {@link WebClientOptions#setThrowExceptionOnFailingStatusCode(boolean)} is set to true.
440      * @throws IOException if an IO problem occurs
441      * @throws MalformedURLException if no URL can be created from the provided string
442      */
443     @SuppressWarnings("unchecked")
444     public <P extends Page> P getPage(final String url) throws IOException, FailingHttpStatusCodeException,
445         MalformedURLException {
446         return (P) getPage(UrlUtils.toUrlUnsafe(url));
447     }
448 
449     /**
450      * Convenient method to load a URL into the current top WebWindow as it would be done
451      * by {@link #getPage(WebWindow, WebRequest)}.
452      * @param url the URL of the new content
453      * @param <P> the page type
454      * @return the new page
455      * @throws FailingHttpStatusCodeException if the server returns a failing status code AND the property
456      *         {@link WebClientOptions#setThrowExceptionOnFailingStatusCode(boolean)} is set to true.
457      * @throws IOException if an IO problem occurs
458      */
459     @SuppressWarnings("unchecked")
460     public <P extends Page> P getPage(final URL url) throws IOException, FailingHttpStatusCodeException {
461         return (P) getPage(getCurrentWindow().getTopWindow(),
462                 new WebRequest(url, getBrowserVersion().getHtmlAcceptHeader()));
463     }
464 
465     /**
466      * Convenient method to load a web request into the current top WebWindow.
467      * @param request the request parameters
468      * @param <P> the page type
469      * @return the new page
470      * @throws FailingHttpStatusCodeException if the server returns a failing status code AND the property
471      *         {@link WebClientOptions#setThrowExceptionOnFailingStatusCode(boolean)} is set to true.
472      * @throws IOException if an IO problem occurs
473      * @see #getPage(WebWindow,WebRequest)
474      */
475     @SuppressWarnings("unchecked")
476     public <P extends Page> P getPage(final WebRequest request) throws IOException,
477         FailingHttpStatusCodeException {
478         return (P) getPage(getCurrentWindow().getTopWindow(), request);
479     }
480 
481     /**
482      * <p>Creates a page based on the specified response and inserts it into the specified window. All page
483      * initialization and event notification is handled here.</p>
484      *
485      * <p>Note that if the page created is an attachment page, and an {@link AttachmentHandler} has been
486      * registered with this client, the page is <b>not</b> loaded into the specified window; in this case,
487      * the page is loaded into a new window, and attachment handling is delegated to the registered
488      * <tt>AttachmentHandler</tt>.</p>
489      *
490      * @param webResponse the response that will be used to create the new page
491      * @param webWindow the window that the new page will be placed within
492      * @throws IOException if an IO error occurs
493      * @throws FailingHttpStatusCodeException if the server returns a failing status code AND the property
494      *         {@link WebClientOptions#setThrowExceptionOnFailingStatusCode(boolean)} is set to true
495      * @return the newly created page
496      * @see #setAttachmentHandler(AttachmentHandler)
497      */
498     public Page loadWebResponseInto(final WebResponse webResponse, final WebWindow webWindow)
499         throws IOException, FailingHttpStatusCodeException {
500 
501         WebAssert.notNull("webResponse", webResponse);
502         WebAssert.notNull("webWindow", webWindow);
503 
504         if (webResponse.getStatusCode() == HttpStatus.SC_NO_CONTENT) {
505             return webWindow.getEnclosedPage();
506         }
507 
508         if (attachmentHandler_ != null && Attachment.isAttachment(webResponse)) {
509             final WebWindow w = openWindow(null, null, webWindow);
510             final Page page = pageCreator_.createPage(webResponse, w);
511             attachmentHandler_.handleAttachment(page);
512             return page;
513         }
514 
515         final Page oldPage = webWindow.getEnclosedPage();
516         if (oldPage != null) {
517             // Remove the old windows before create new ones.
518             oldPage.cleanUp();
519         }
520         Page newPage = null;
521         if (windows_.contains(webWindow) || getBrowserVersion().hasFeature(WINDOW_EXECUTE_EVENTS)) {
522             newPage = pageCreator_.createPage(webResponse, webWindow);
523 
524             if (windows_.contains(webWindow)) {
525                 fireWindowContentChanged(new WebWindowEvent(webWindow, WebWindowEvent.CHANGE, oldPage, newPage));
526 
527                 // The page being loaded may already have been replaced by another page via JavaScript code.
528                 if (webWindow.getEnclosedPage() == newPage) {
529                     newPage.initialize();
530                     // hack: onload should be fired the same way for all type of pages
531                     // here is a hack to handle non HTML pages
532                     if (webWindow instanceof FrameWindow && !newPage.isHtmlPage()) {
533                         final FrameWindow fw = (FrameWindow) webWindow;
534                         final BaseFrameElement frame = fw.getFrameElement();
535                         if (frame.hasEventHandlers("onload")) {
536                             if (LOG.isDebugEnabled()) {
537                                 LOG.debug("Executing onload handler for " + frame);
538                             }
539                             final Event event = new Event(frame, Event.TYPE_LOAD);
540                             ((Node) frame.getScriptableObject()).executeEventLocally(event);
541                         }
542                     }
543                 }
544             }
545         }
546         return newPage;
547     }
548 
549     /**
550      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span>
551      *
552      * <p>Logs the response's content if its status code indicates a request failure and
553      * {@link WebClientOptions#isPrintContentOnFailingStatusCode()} returns {@code true}.
554      *
555      * @param webResponse the response whose content may be logged
556      */
557     public void printContentIfNecessary(final WebResponse webResponse) {
558         final String contentType = webResponse.getContentType();
559         final int statusCode = webResponse.getStatusCode();
560         final boolean successful = statusCode >= HttpStatus.SC_OK && statusCode < HttpStatus.SC_MULTIPLE_CHOICES;
561         if (getOptions().isPrintContentOnFailingStatusCode() && !successful) {
562             LOG.info("statusCode=[" + statusCode + "] contentType=[" + contentType + "]");
563             LOG.info(webResponse.getContentAsString());
564         }
565     }
566 
567     /**
568      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span>
569      *
570      * <p>Throws a {@link FailingHttpStatusCodeException} if the request's status code indicates a request
571      * failure and {@link WebClientOptions#isThrowExceptionOnFailingStatusCode()} returns {@code true}.
572      *
573      * @param webResponse the response which may trigger a {@link FailingHttpStatusCodeException}
574      */
575     public void throwFailingHttpStatusCodeExceptionIfNecessary(final WebResponse webResponse) {
576         final int statusCode = webResponse.getStatusCode();
577         final boolean successful = (statusCode >= HttpStatus.SC_OK && statusCode < HttpStatus.SC_MULTIPLE_CHOICES)
578             || statusCode == HttpStatus.SC_USE_PROXY
579             || statusCode == HttpStatus.SC_NOT_MODIFIED;
580         if (getOptions().isThrowExceptionOnFailingStatusCode() && !successful) {
581             throw new FailingHttpStatusCodeException(webResponse);
582         }
583     }
584 
585     /**
586      * Adds a header which will be sent with EVERY request from this client.
587      * @param name the name of the header to add
588      * @param value the value of the header to add
589      * @see #removeRequestHeader(String)
590      */
591     public void addRequestHeader(final String name, final String value) {
592         if ("cookie".equalsIgnoreCase(name)) {
593             throw new IllegalArgumentException("Do not add 'Cookie' header, use .getCookieManager() instead");
594         }
595         requestHeaders_.put(name, value);
596     }
597 
598     /**
599      * Removes a header from being sent with EVERY request from this client.
600      * @param name the name of the header to remove
601      * @see #addRequestHeader
602      */
603     public void removeRequestHeader(final String name) {
604         requestHeaders_.remove(name);
605     }
606 
607     /**
608      * Sets the credentials provider that will provide authentication information when
609      * trying to access protected information on a web server. This information is
610      * required when the server is using Basic HTTP authentication, NTLM authentication,
611      * or Digest authentication.
612      * @param credentialsProvider the new credentials provider to use to authenticate
613      */
614     public void setCredentialsProvider(final CredentialsProvider credentialsProvider) {
615         WebAssert.notNull("credentialsProvider", credentialsProvider);
616         credentialsProvider_ = credentialsProvider;
617     }
618 
619     /**
620      * Returns the credentials provider for this client instance. By default, this
621      * method returns an instance of {@link DefaultCredentialsProvider}.
622      * @return the credentials provider for this client instance
623      */
624     public CredentialsProvider getCredentialsProvider() {
625         return credentialsProvider_;
626     }
627 
628     /**
629      * This method is intended for testing only - use at your own risk.
630      * @return the current JavaScript engine (never {@code null})
631      */
632     public AbstractJavaScriptEngine<?> getJavaScriptEngine() {
633         return scriptEngine_;
634     }
635 
636     /**
637      * This method is intended for testing only - use at your own risk.
638      *
639      * @param engine the new script engine to use
640      */
641     public void setJavaScriptEngine(final AbstractJavaScriptEngine<?> engine) {
642         if (engine == null) {
643             throw new IllegalArgumentException("Can't set JavaScriptEngine to null");
644         }
645         scriptEngine_ = engine;
646     }
647 
648     /**
649      * Returns the cookie manager used by this web client.
650      * @return the cookie manager used by this web client
651      */
652     public CookieManager getCookieManager() {
653         return cookieManager_;
654     }
655 
656     /**
657      * Sets the cookie manager used by this web client.
658      * @param cookieManager the cookie manager used by this web client
659      */
660     public void setCookieManager(final CookieManager cookieManager) {
661         WebAssert.notNull("cookieManager", cookieManager);
662         cookieManager_ = cookieManager;
663     }
664 
665     /**
666      * Sets the alert handler for this webclient.
667      * @param alertHandler the new alerthandler or null if none is specified
668      */
669     public void setAlertHandler(final AlertHandler alertHandler) {
670         alertHandler_ = alertHandler;
671     }
672 
673     /**
674      * Returns the alert handler for this webclient.
675      * @return the alert handler or null if one hasn't been set
676      */
677     public AlertHandler getAlertHandler() {
678         return alertHandler_;
679     }
680 
681     /**
682      * Sets the handler that will be executed when the JavaScript method Window.confirm() is called.
683      * @param handler the new handler or null if no handler is to be used
684      */
685     public void setConfirmHandler(final ConfirmHandler handler) {
686         confirmHandler_ = handler;
687     }
688 
689     /**
690      * Returns the confirm handler.
691      * @return the confirm handler or null if one hasn't been set
692      */
693     public ConfirmHandler getConfirmHandler() {
694         return confirmHandler_;
695     }
696 
697     /**
698      * Sets the handler that will be executed when the JavaScript method Window.prompt() is called.
699      * @param handler the new handler or null if no handler is to be used
700      */
701     public void setPromptHandler(final PromptHandler handler) {
702         promptHandler_ = handler;
703     }
704 
705     /**
706      * Returns the prompt handler.
707      * @return the prompt handler or null if one hasn't been set
708      */
709     public PromptHandler getPromptHandler() {
710         return promptHandler_;
711     }
712 
713     /**
714      * Sets the status handler for this webclient.
715      * @param statusHandler the new status handler or null if none is specified
716      */
717     public void setStatusHandler(final StatusHandler statusHandler) {
718         statusHandler_ = statusHandler;
719     }
720 
721     /**
722      * Returns the status handler for this webclient.
723      * @return the status handler or null if one hasn't been set
724      */
725     public StatusHandler getStatusHandler() {
726         return statusHandler_;
727     }
728 
729     /**
730      * Sets the javascript error listener for this webclient.
731      * When setting to null, the {@link DefaultJavaScriptErrorListener} is used.
732      * @param javaScriptErrorListener the new JavaScriptErrorListener or null if none is specified
733      */
734     public void setJavaScriptErrorListener(final JavaScriptErrorListener javaScriptErrorListener) {
735         if (javaScriptErrorListener == null) {
736             javaScriptErrorListener_ = new DefaultJavaScriptErrorListener();
737         }
738         else {
739             javaScriptErrorListener_ = javaScriptErrorListener;
740         }
741     }
742 
743     /**
744      * Returns the javascript error listener for this webclient.
745      * @return the javascript error listener or null if one hasn't been set
746      */
747     public JavaScriptErrorListener getJavaScriptErrorListener() {
748         return javaScriptErrorListener_;
749     }
750 
751     /**
752      * Returns the current browser version.
753      * @return the current browser version
754      */
755     public BrowserVersion getBrowserVersion() {
756         return browserVersion_;
757     }
758 
759     /**
760      * Returns the "current" window for this client. This window (or its top window) will be used
761      * when <tt>getPage(...)</tt> is called without specifying a window.
762      * @return the "current" window for this client
763      */
764     public WebWindow getCurrentWindow() {
765         return currentWindow_;
766     }
767 
768     /**
769      * Sets the "current" window for this client. This is the window that will be used when
770      * <tt>getPage(...)</tt> is called without specifying a window.
771      * @param window the new "current" window for this client
772      */
773     public void setCurrentWindow(final WebWindow window) {
774         WebAssert.notNull("window", window);
775         if (currentWindow_ == window) {
776             return;
777         }
778         // onBlur event is triggered for focused element of old current window
779         if (currentWindow_ != null && !currentWindow_.isClosed()) {
780             final Page enclosedPage = currentWindow_.getEnclosedPage();
781             if (enclosedPage != null && enclosedPage.isHtmlPage()) {
782                 final DomElement focusedElement = ((HtmlPage) enclosedPage).getFocusedElement();
783                 if (focusedElement != null) {
784                     focusedElement.fireEvent(Event.TYPE_BLUR);
785                 }
786             }
787         }
788         currentWindow_ = window;
789 
790         // when marking an iframe window as current we have no need to move the focus
791         final boolean isIFrame = currentWindow_ instanceof FrameWindow
792                 && ((FrameWindow) currentWindow_).getFrameElement() instanceof HtmlInlineFrame;
793         if (!isIFrame) {
794             //1. activeElement becomes focused element for new current window
795             //2. onFocus event is triggered for focusedElement of new current window
796             final Page enclosedPage = currentWindow_.getEnclosedPage();
797             if (enclosedPage != null && enclosedPage.isHtmlPage()) {
798                 final Object jsWindow = currentWindow_.getScriptableObject();
799                 if (jsWindow instanceof Window) {
800                     final HTMLElement activeElement =
801                             ((HTMLDocument) ((Window) jsWindow).getDocument()).getActiveElement();
802                     if (activeElement != null) {
803                         ((HtmlPage) enclosedPage).setFocusedElement(activeElement.getDomNodeOrDie(), true);
804                     }
805                 }
806             }
807         }
808     }
809 
810     /**
811      * Adds a listener for {@link WebWindowEvent}s. All events from all windows associated with this
812      * client will be sent to the specified listener.
813      * @param listener a listener
814      */
815     public void addWebWindowListener(final WebWindowListener listener) {
816         WebAssert.notNull("listener", listener);
817         webWindowListeners_.add(listener);
818     }
819 
820     /**
821      * Removes a listener for {@link WebWindowEvent}s.
822      * @param listener a listener
823      */
824     public void removeWebWindowListener(final WebWindowListener listener) {
825         WebAssert.notNull("listener", listener);
826         webWindowListeners_.remove(listener);
827     }
828 
829     private void fireWindowContentChanged(final WebWindowEvent event) {
830         for (final WebWindowListener listener : new ArrayList<>(webWindowListeners_)) {
831             listener.webWindowContentChanged(event);
832         }
833     }
834 
835     private void fireWindowOpened(final WebWindowEvent event) {
836         for (final WebWindowListener listener : new ArrayList<>(webWindowListeners_)) {
837             listener.webWindowOpened(event);
838         }
839     }
840 
841     private void fireWindowClosed(final WebWindowEvent event) {
842         for (final WebWindowListener listener : new ArrayList<>(webWindowListeners_)) {
843             listener.webWindowClosed(event);
844         }
845     }
846 
847     /**
848      * Open a new window with the specified name. If the URL is non-null then attempt to load
849      * a page from that location and put it in the new window.
850      *
851      * @param url the URL to load content from or null if no content is to be loaded
852      * @param windowName the name of the new window
853      * @return the new window
854      */
855     public WebWindow openWindow(final URL url, final String windowName) {
856         WebAssert.notNull("windowName", windowName);
857         return openWindow(url, windowName, getCurrentWindow());
858     }
859 
860     /**
861      * Open a new window with the specified name. If the URL is non-null then attempt to load
862      * a page from that location and put it in the new window.
863      *
864      * @param url the URL to load content from or null if no content is to be loaded
865      * @param windowName the name of the new window
866      * @param opener the web window that is calling openWindow
867      * @return the new window
868      */
869     public WebWindow openWindow(final URL url, final String windowName, final WebWindow opener) {
870         final WebWindow window = openTargetWindow(opener, windowName, TARGET_BLANK);
871         final HtmlPage openerPage = (HtmlPage) opener.getEnclosedPage();
872         if (url != null) {
873             try {
874                 final WebRequest request = new WebRequest(url, getBrowserVersion().getHtmlAcceptHeader());
875                 if (getBrowserVersion().hasFeature(DIALOGWINDOW_REFERER)
876                         && openerPage != null) {
877                     final String referer = openerPage.getUrl().toExternalForm();
878                     request.setAdditionalHeader("Referer", referer);
879                 }
880                 getPage(window, request);
881             }
882             catch (final IOException e) {
883                 LOG.error("Error loading content into window", e);
884             }
885         }
886         else {
887             initializeEmptyWindow(window);
888         }
889         return window;
890     }
891 
892     /**
893      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
894      *
895      * Open the window with the specified name. The name may be a special
896      * target name of _self, _parent, _top, or _blank. An empty or null
897      * name is set to the default. The special target names are relative to
898      * the opener window.
899      *
900      * @param opener the web window that is calling openWindow
901      * @param windowName the name of the new window
902      * @param defaultName the default target if no name is given
903      * @return the new window
904      */
905     public WebWindow openTargetWindow(
906             final WebWindow opener, final String windowName, final String defaultName) {
907 
908         WebAssert.notNull("opener", opener);
909         WebAssert.notNull("defaultName", defaultName);
910 
911         String windowToOpen = windowName;
912         if (windowToOpen == null || windowToOpen.isEmpty()) {
913             windowToOpen = defaultName;
914         }
915 
916         WebWindow webWindow = resolveWindow(opener, windowToOpen);
917 
918         if (webWindow == null) {
919             if (TARGET_BLANK.equals(windowToOpen)) {
920                 windowToOpen = "";
921             }
922             webWindow = new TopLevelWindow(windowToOpen, this);
923             fireWindowOpened(new WebWindowEvent(webWindow, WebWindowEvent.OPEN, null, null));
924         }
925 
926         if (webWindow instanceof TopLevelWindow && webWindow != opener.getTopWindow()) {
927             ((TopLevelWindow) webWindow).setOpener(opener);
928         }
929 
930         return webWindow;
931     }
932 
933     private WebWindow resolveWindow(final WebWindow opener, final String name) {
934         if (name == null || name.isEmpty() || TARGET_SELF.equals(name)) {
935             return opener;
936         }
937 
938         if (TARGET_PARENT.equals(name)) {
939             return opener.getParentWindow();
940         }
941 
942         if (TARGET_TOP.equals(name)) {
943             return opener.getTopWindow();
944         }
945 
946         if (TARGET_BLANK.equals(name)) {
947             return null;
948         }
949 
950         // first search for frame windows inside our window hierarchy
951         WebWindow window = opener;
952         while (true) {
953             final Page page = window.getEnclosedPage();
954             if (page != null && page.isHtmlPage()) {
955                 try {
956                     final FrameWindow frame = ((HtmlPage) page).getFrameByName(name);
957                     final ScriptableObject scriptable = frame.getFrameElement().getScriptableObject();
958                     if (scriptable instanceof HTMLIFrameElement) {
959                         ((HTMLIFrameElement) scriptable).onRefresh();
960                     }
961                     return frame;
962                 }
963                 catch (final ElementNotFoundException e) {
964                     // Fall through
965                 }
966             }
967 
968             if (window == window.getParentWindow()) {
969                 // TODO: should getParentWindow() return null on top windows?
970                 break;
971             }
972             window = window.getParentWindow();
973         }
974 
975         try {
976             return getWebWindowByName(name);
977         }
978         catch (final WebWindowNotFoundException e) {
979             // Fall through - a new window will be created below
980         }
981         return null;
982     }
983 
984     /**
985      * <p><span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span></p>
986      *
987      * Opens a new dialog window.
988      * @param url the URL of the document to load and display
989      * @param opener the web window that is opening the dialog
990      * @param dialogArguments the object to make available inside the dialog via <tt>window.dialogArguments</tt>
991      * @return the new dialog window
992      * @throws IOException if there is an IO error
993      */
994     public DialogWindow openDialogWindow(final URL url, final WebWindow opener, final Object dialogArguments)
995         throws IOException {
996 
997         WebAssert.notNull("url", url);
998         WebAssert.notNull("opener", opener);
999 
1000         final DialogWindow window = new DialogWindow(this, dialogArguments);
1001         fireWindowOpened(new WebWindowEvent(window, WebWindowEvent.OPEN, null, null));
1002 
1003         final HtmlPage openerPage = (HtmlPage) opener.getEnclosedPage();
1004         final WebRequest request = new WebRequest(url, getBrowserVersion().getHtmlAcceptHeader());
1005         if (getBrowserVersion().hasFeature(DIALOGWINDOW_REFERER) && openerPage != null) {
1006             final String referer = openerPage.getUrl().toExternalForm();
1007             request.setAdditionalHeader("Referer", referer);
1008         }
1009 
1010         getPage(window, request);
1011 
1012         return window;
1013     }
1014 
1015     /**
1016      * Sets the object that will be used to create pages. Set this if you want
1017      * to customize the type of page that is returned for a given content type.
1018      *
1019      * @param pageCreator the new page creator
1020      */
1021     public void setPageCreator(final PageCreator pageCreator) {
1022         WebAssert.notNull("pageCreator", pageCreator);
1023         pageCreator_ = pageCreator;
1024     }
1025 
1026     /**
1027      * Returns the current page creator.
1028      *
1029      * @return the page creator
1030      */
1031     public PageCreator getPageCreator() {
1032         return pageCreator_;
1033     }
1034 
1035     /**
1036      * Returns the first {@link WebWindow} that matches the specified name.
1037      *
1038      * @param name the name to search for
1039      * @return the {@link WebWindow} with the specified name
1040      * @throws WebWindowNotFoundException if the {@link WebWindow} can't be found
1041      * @see #getWebWindows()
1042      * @see #getTopLevelWindows()
1043      */
1044     public WebWindow getWebWindowByName(final String name) throws WebWindowNotFoundException {
1045         WebAssert.notNull("name", name);
1046 
1047         for (final WebWindow webWindow : windows_) {
1048             if (name.equals(webWindow.getName())) {
1049                 return webWindow;
1050             }
1051         }
1052 
1053         throw new WebWindowNotFoundException(name);
1054     }
1055 
1056     /**
1057      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
1058      *
1059      * Initializes a new web window for JavaScript.
1060      * @param webWindow the new WebWindow
1061      */
1062     public void initialize(final WebWindow webWindow) {
1063         WebAssert.notNull("webWindow", webWindow);
1064         scriptEngine_.initialize(webWindow);
1065     }
1066 
1067     /**
1068      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
1069      *
1070      * Initializes a new page for JavaScript.
1071      * @param newPage the new page
1072      */
1073     public void initialize(final Page newPage) {
1074         WebAssert.notNull("newPage", newPage);
1075         final WebWindow webWindow = newPage.getEnclosingWindow();
1076         if (webWindow.getScriptableObject() instanceof Window) {
1077             ((Window) webWindow.getScriptableObject()).initialize(newPage);
1078         }
1079     }
1080 
1081     /**
1082      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
1083      *
1084      * Initializes a new empty window for JavaScript.
1085      *
1086      * @param webWindow the new WebWindow
1087      */
1088     public void initializeEmptyWindow(final WebWindow webWindow) {
1089         WebAssert.notNull("webWindow", webWindow);
1090         initialize(webWindow);
1091         ((Window) webWindow.getScriptableObject()).initialize();
1092     }
1093 
1094     /**
1095      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
1096      *
1097      * Adds a new window to the list of available windows.
1098      *
1099      * @param webWindow the new WebWindow
1100      */
1101     public void registerWebWindow(final WebWindow webWindow) {
1102         WebAssert.notNull("webWindow", webWindow);
1103         windows_.add(webWindow);
1104         // register JobManager here but don't deregister in deregisterWebWindow as it can live longer
1105         jobManagers_.add(new WeakReference<>(webWindow.getJobManager()));
1106     }
1107 
1108     /**
1109      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
1110      *
1111      * Removes a window from the list of available windows.
1112      *
1113      * @param webWindow the window to remove
1114      */
1115     public void deregisterWebWindow(final WebWindow webWindow) {
1116         WebAssert.notNull("webWindow", webWindow);
1117         if (windows_.remove(webWindow)) {
1118             fireWindowClosed(new WebWindowEvent(webWindow, WebWindowEvent.CLOSE, webWindow.getEnclosedPage(), null));
1119         }
1120     }
1121 
1122     /**
1123      * Expands a relative URL relative to the specified base. In most situations
1124      * this is the same as <code>new URL(baseUrl, relativeUrl)</code> but
1125      * there are some cases that URL doesn't handle correctly. See
1126      * <a href="http://www.faqs.org/rfcs/rfc1808.html">RFC1808</a>
1127      * regarding Relative Uniform Resource Locators for more information.
1128      *
1129      * @param baseUrl the base URL
1130      * @param relativeUrl the relative URL
1131      * @return the expansion of the specified base and relative URLs
1132      * @throws MalformedURLException if an error occurred when creating a URL object
1133      */
1134     public static URL expandUrl(final URL baseUrl, final String relativeUrl) throws MalformedURLException {
1135         final String newUrl = UrlUtils.resolveUrl(baseUrl, relativeUrl);
1136         return UrlUtils.toUrlUnsafe(newUrl);
1137     }
1138 
1139     private WebResponse makeWebResponseForDataUrl(final WebRequest webRequest) throws IOException {
1140         final URL url = webRequest.getUrl();
1141         final List<NameValuePair> responseHeaders = new ArrayList<>();
1142         final DataUrlDecoder decoder;
1143         try {
1144             decoder = DataUrlDecoder.decode(url);
1145         }
1146         catch (final DecoderException e) {
1147             throw new IOException(e.getMessage());
1148         }
1149         responseHeaders.add(new NameValuePair("content-type",
1150             decoder.getMediaType() + ";charset=" + decoder.getCharset()));
1151         final DownloadedContent downloadedContent =
1152                 HttpWebConnection.downloadContent(url.openStream(), getOptions().getMaxInMemory());
1153         final WebResponseData data = new WebResponseData(downloadedContent, 200, "OK", responseHeaders);
1154         return new WebResponse(data, url, webRequest.getHttpMethod(), 0);
1155     }
1156 
1157     private static WebResponse makeWebResponseForAboutUrl(final URL url) {
1158         final String urlWithoutQuery = StringUtils.substringBefore(url.toExternalForm(), "?");
1159         if (!"blank".equalsIgnoreCase(StringUtils.substringAfter(urlWithoutQuery, WebClient.ABOUT_SCHEME))) {
1160             throw new IllegalArgumentException(url + " is not supported, only about:blank is supported now.");
1161         }
1162         return new StringWebResponse("", URL_ABOUT_BLANK);
1163     }
1164 
1165     /**
1166      * Builds a WebResponse for a file URL.
1167      * This first implementation is basic.
1168      * It assumes that the file contains an HTML page encoded with the specified encoding.
1169      * @param url the file URL
1170      * @param charset encoding to use
1171      * @return the web response
1172      * @throws IOException if an IO problem occurs
1173      */
1174     private WebResponse makeWebResponseForFileUrl(final WebRequest webRequest) throws IOException {
1175         URL cleanUrl = webRequest.getUrl();
1176         if (cleanUrl.getQuery() != null) {
1177             // Get rid of the query portion before trying to load the file.
1178             cleanUrl = UrlUtils.getUrlWithNewQuery(cleanUrl, null);
1179         }
1180         if (cleanUrl.getRef() != null) {
1181             // Get rid of the ref portion before trying to load the file.
1182             cleanUrl = UrlUtils.getUrlWithNewRef(cleanUrl, null);
1183         }
1184 
1185         String fileUrl = cleanUrl.toExternalForm();
1186         fileUrl = URLDecoder.decode(fileUrl, UTF_8.name());
1187         final File file = new File(fileUrl.substring(5));
1188         if (!file.exists()) {
1189             // construct 404
1190             final List<NameValuePair> compiledHeaders = new ArrayList<>();
1191             compiledHeaders.add(new NameValuePair("Content-Type", "text/html"));
1192             final WebResponseData responseData =
1193                 new WebResponseData(
1194                     TextUtil.stringToByteArray("File: " + file.getAbsolutePath(), UTF_8),
1195                     404, "Not Found", compiledHeaders);
1196             return new WebResponse(responseData, webRequest, 0);
1197         }
1198 
1199         final String contentType = guessContentType(file);
1200 
1201         final DownloadedContent content = new DownloadedContent.OnFile(file, false);
1202         final List<NameValuePair> compiledHeaders = new ArrayList<>();
1203         compiledHeaders.add(new NameValuePair("Content-Type", contentType));
1204         final WebResponseData responseData = new WebResponseData(content, 200, "OK", compiledHeaders);
1205         return new WebResponse(responseData, webRequest, 0);
1206     }
1207 
1208     /**
1209      * Tries to guess the content type of the file.<br>
1210      * This utility could be located in an helper class but we can compare this functionality
1211      * for instance with the "Helper Applications" settings of Mozilla and therefore see it as a
1212      * property of the "browser".
1213      * @param file the file
1214      * @return "application/octet-stream" if nothing could be guessed
1215      */
1216     public String guessContentType(final File file) {
1217         final String fileName = file.getName();
1218         if (fileName.endsWith(".xhtml")) {
1219             // Java's mime type map returns application/xml in JDK8.
1220             return "application/xhtml+xml";
1221         }
1222 
1223         // Java's mime type map does not know these in JDK8.
1224         if (fileName.endsWith(".js")) {
1225             return "text/javascript";
1226         }
1227         if (fileName.toLowerCase(Locale.ROOT).endsWith(".css")) {
1228             return "text/css";
1229         }
1230 
1231         String contentType = URLConnection.guessContentTypeFromName(fileName);
1232         if (contentType == null) {
1233             try (InputStream inputStream = new BufferedInputStream(new FileInputStream(file))) {
1234                 contentType = URLConnection.guessContentTypeFromStream(inputStream);
1235             }
1236             catch (final IOException e) {
1237                 // Ignore silently.
1238             }
1239         }
1240         if (contentType == null) {
1241             contentType = "application/octet-stream";
1242         }
1243         return contentType;
1244     }
1245 
1246     private WebResponse makeWebResponseForJavaScriptUrl(final WebWindow webWindow, final URL url,
1247         final Charset charset) throws FailingHttpStatusCodeException, IOException {
1248 
1249         HtmlPage page = null;
1250         if (webWindow instanceof FrameWindow) {
1251             final FrameWindow frameWindow = (FrameWindow) webWindow;
1252             page = (HtmlPage) frameWindow.getEnclosedPage();
1253         }
1254         else {
1255             Page currentPage = webWindow.getEnclosedPage();
1256             if (currentPage == null) {
1257                 // Starting with a JavaScript URL; quickly fill an "about:blank".
1258                 currentPage = getPage(webWindow, new WebRequest(WebClient.URL_ABOUT_BLANK));
1259             }
1260             else if (currentPage instanceof HtmlPage) {
1261                 page = (HtmlPage) currentPage;
1262             }
1263         }
1264 
1265         if (page == null) {
1266             page = getPage(webWindow, new WebRequest(WebClient.URL_ABOUT_BLANK));
1267         }
1268         final ScriptResult r = page.executeJavaScript(url.toExternalForm(), "JavaScript URL", 1);
1269         if (r.getJavaScriptResult() == null || ScriptResult.isUndefined(r)) {
1270             // No new WebResponse to produce.
1271             return webWindow.getEnclosedPage().getWebResponse();
1272         }
1273 
1274         final String contentString = r.getJavaScriptResult().toString();
1275         final StringWebResponse response = new StringWebResponse(contentString, charset, url);
1276         response.setFromJavascript(true);
1277         return response;
1278     }
1279 
1280     /**
1281      * Loads a {@link WebResponse} from the server.
1282      * @param webRequest the request
1283      * @throws IOException if an IO problem occurs
1284      * @return the WebResponse
1285      */
1286     public WebResponse loadWebResponse(final WebRequest webRequest) throws IOException {
1287         switch (webRequest.getUrl().getProtocol()) {
1288             case "about":
1289                 return makeWebResponseForAboutUrl(webRequest.getUrl());
1290 
1291             case "file":
1292                 return makeWebResponseForFileUrl(webRequest);
1293 
1294             case "data":
1295                 return makeWebResponseForDataUrl(webRequest);
1296 
1297             default:
1298                 return loadWebResponseFromWebConnection(webRequest, ALLOWED_REDIRECTIONS_SAME_URL);
1299         }
1300     }
1301 
1302     /**
1303      * Loads a {@link WebResponse} from the server through the WebConnection.
1304      * @param webRequest the request
1305      * @param allowedRedirects the number of allowed redirects remaining
1306      * @throws IOException if an IO problem occurs
1307      * @return the resultant {@link WebResponse}
1308      */
1309     private WebResponse loadWebResponseFromWebConnection(final WebRequest webRequest,
1310         final int allowedRedirects) throws IOException {
1311 
1312         URL url = webRequest.getUrl();
1313         final HttpMethod method = webRequest.getHttpMethod();
1314         final List<NameValuePair> parameters = webRequest.getRequestParameters();
1315 
1316         WebAssert.notNull("url", url);
1317         WebAssert.notNull("method", method);
1318         WebAssert.notNull("parameters", parameters);
1319 
1320         url = UrlUtils.encodeUrl(url, getBrowserVersion().hasFeature(URL_MINIMAL_QUERY_ENCODING),
1321                 webRequest.getCharset());
1322         webRequest.setUrl(url);
1323 
1324         if (LOG.isDebugEnabled()) {
1325             LOG.debug("Load response for " + method + " " + url.toExternalForm());
1326         }
1327 
1328         // If the request settings don't specify a custom proxy, use the default client proxy...
1329         if (webRequest.getProxyHost() == null) {
1330             final ProxyConfig proxyConfig = getOptions().getProxyConfig();
1331             if (proxyConfig.getProxyAutoConfigUrl() != null) {
1332                 if (!UrlUtils.sameFile(new URL(proxyConfig.getProxyAutoConfigUrl()), url)) {
1333                     String content = proxyConfig.getProxyAutoConfigContent();
1334                     if (content == null) {
1335                         content = getPage(proxyConfig.getProxyAutoConfigUrl())
1336                             .getWebResponse().getContentAsString();
1337                         proxyConfig.setProxyAutoConfigContent(content);
1338                     }
1339                     final String allValue = ProxyAutoConfig.evaluate(content, url);
1340                     if (LOG.isDebugEnabled()) {
1341                         LOG.debug("Proxy Auto-Config: value '" + allValue + "' for URL " + url);
1342                     }
1343                     String value = allValue.split(";")[0].trim();
1344                     if (value.startsWith("PROXY")) {
1345                         value = value.substring(6);
1346                         final int colonIndex = value.indexOf(':');
1347                         webRequest.setSocksProxy(false);
1348                         webRequest.setProxyHost(value.substring(0, colonIndex));
1349                         webRequest.setProxyPort(Integer.parseInt(value.substring(colonIndex + 1)));
1350                     }
1351                     else if (value.startsWith("SOCKS")) {
1352                         value = value.substring(6);
1353                         final int colonIndex = value.indexOf(':');
1354                         webRequest.setSocksProxy(true);
1355                         webRequest.setProxyHost(value.substring(0, colonIndex));
1356                         webRequest.setProxyPort(Integer.parseInt(value.substring(colonIndex + 1)));
1357                     }
1358                 }
1359             }
1360             // ...unless the host needs to bypass the configured client proxy!
1361             else if (!proxyConfig.shouldBypassProxy(webRequest.getUrl().getHost())) {
1362                 webRequest.setProxyHost(proxyConfig.getProxyHost());
1363                 webRequest.setProxyPort(proxyConfig.getProxyPort());
1364                 webRequest.setSocksProxy(proxyConfig.isSocksProxy());
1365             }
1366         }
1367 
1368         // Add the headers that are sent with every request.
1369         addDefaultHeaders(webRequest);
1370 
1371         // Retrieve the response, either from the cache or from the server.
1372         final WebResponse fromCache = getCache().getCachedResponse(webRequest);
1373         final WebResponse webResponse;
1374         if (fromCache != null) {
1375             webResponse = new WebResponseFromCache(fromCache, webRequest);
1376         }
1377         else {
1378             try {
1379                 webResponse = getWebConnection().getResponse(webRequest);
1380             }
1381             catch (final NoHttpResponseException e) {
1382                 return new WebResponse(responseDataNoHttpResponse_, webRequest, 0);
1383             }
1384             getCache().cacheIfPossible(webRequest, webResponse, null);
1385         }
1386 
1387         // Continue according to the HTTP status code.
1388         final int status = webResponse.getStatusCode();
1389         if (status == HttpStatus.SC_USE_PROXY) {
1390             getIncorrectnessListener().notify("Ignoring HTTP status code [305] 'Use Proxy'", this);
1391         }
1392         else if (status >= HttpStatus.SC_MOVED_PERMANENTLY
1393             && status <= (getBrowserVersion().hasFeature(HTTP_REDIRECT_308) ? 308 : 307)
1394             && status != HttpStatus.SC_NOT_MODIFIED
1395             && getOptions().isRedirectEnabled()) {
1396 
1397             final URL newUrl;
1398             String locationString = null;
1399             try {
1400                 locationString = webResponse.getResponseHeaderValue("Location");
1401                 if (locationString == null) {
1402                     return webResponse;
1403                 }
1404                 newUrl = expandUrl(url, locationString);
1405             }
1406             catch (final MalformedURLException e) {
1407                 getIncorrectnessListener().notify("Got a redirect status code [" + status + " "
1408                     + webResponse.getStatusMessage()
1409                     + "] but the location is not a valid URL [" + locationString
1410                     + "]. Skipping redirection processing.", this);
1411                 return webResponse;
1412             }
1413 
1414             if (LOG.isDebugEnabled()) {
1415                 LOG.debug("Got a redirect status code [" + status + "] new location = [" + locationString + "]");
1416             }
1417 
1418             if (allowedRedirects == 0) {
1419                 throw new FailingHttpStatusCodeException("Too much redirect for "
1420                     + webResponse.getWebRequest().getUrl(), webResponse);
1421             }
1422             else if (status == HttpStatus.SC_MOVED_PERMANENTLY
1423                     || status == HttpStatus.SC_MOVED_TEMPORARILY
1424                     || status == HttpStatus.SC_SEE_OTHER) {
1425                 final WebRequest wrs = new WebRequest(newUrl, HttpMethod.GET);
1426                 for (final Map.Entry<String, String> entry : webRequest.getAdditionalHeaders().entrySet()) {
1427                     wrs.setAdditionalHeader(entry.getKey(), entry.getValue());
1428                 }
1429                 return loadWebResponseFromWebConnection(wrs, allowedRedirects - 1);
1430             }
1431             else if (status == HttpStatus.SC_TEMPORARY_REDIRECT
1432                         || status == 308) {
1433                 final WebRequest wrs = new WebRequest(newUrl, webRequest.getHttpMethod());
1434                 wrs.setRequestParameters(parameters);
1435                 for (final Map.Entry<String, String> entry : webRequest.getAdditionalHeaders().entrySet()) {
1436                     wrs.setAdditionalHeader(entry.getKey(), entry.getValue());
1437                 }
1438                 return loadWebResponseFromWebConnection(wrs, allowedRedirects - 1);
1439             }
1440         }
1441 
1442         return webResponse;
1443     }
1444 
1445     /**
1446      * Adds the headers that are sent with every request to the specified {@link WebRequest} instance.
1447      * @param wrs the <tt>WebRequestSettings</tt> instance to modify
1448      */
1449     private void addDefaultHeaders(final WebRequest wrs) {
1450         // Add standard HtmlUnit headers.
1451         if (!wrs.isAdditionalHeader("Accept-Language")) {
1452             wrs.setAdditionalHeader("Accept-Language", getBrowserVersion().getBrowserLanguage());
1453         }
1454         // Add user-specified headers last so that they can override HtmlUnit defaults.
1455         wrs.getAdditionalHeaders().putAll(requestHeaders_);
1456     }
1457 
1458     /**
1459      * Returns an immutable list of open web windows (whether they are top level windows or not).
1460      * This is a snapshot; future changes are not reflected by this list.
1461      *
1462      * @return an immutable list of open web windows (whether they are top level windows or not)
1463      * @see #getWebWindowByName(String)
1464      * @see #getTopLevelWindows()
1465      */
1466     public List<WebWindow> getWebWindows() {
1467         return Collections.unmodifiableList(new ArrayList<>(windows_));
1468     }
1469 
1470     /**
1471      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
1472      *
1473      * Returns true if the list of WebWindows contains the provided one.
1474      * This method is there to improve the performance of some internal checks because
1475      * calling getWebWindows().contains(.) creates some objects without any need.
1476      *
1477      * @param webWindow the window to check
1478      * @return true or false
1479      */
1480     public boolean containsWebWindow(final WebWindow webWindow) {
1481         return windows_.contains(webWindow);
1482     }
1483 
1484     /**
1485      * Returns an immutable list of open top level windows.
1486      * This is a snapshot; future changes are not reflected by this list.
1487      *
1488      * @return an immutable list of open top level windows
1489      * @see #getWebWindowByName(String)
1490      * @see #getWebWindows()
1491      */
1492     public List<TopLevelWindow> getTopLevelWindows() {
1493         return Collections.unmodifiableList(new ArrayList<>(topLevelWindows_));
1494     }
1495 
1496     /**
1497      * Sets the handler to be used whenever a refresh is triggered. Refer
1498      * to the documentation for {@link RefreshHandler} for more details.
1499      * @param handler the new handler
1500      */
1501     public void setRefreshHandler(final RefreshHandler handler) {
1502         if (handler == null) {
1503             refreshHandler_ = new NiceRefreshHandler(2);
1504         }
1505         else {
1506             refreshHandler_ = handler;
1507         }
1508     }
1509 
1510     /**
1511      * Returns the current refresh handler.
1512      * The default refresh handler is a {@link NiceRefreshHandler NiceRefreshHandler(2)} (since HtmlUnit-2.12).
1513      * @return the current RefreshHandler
1514      */
1515     public RefreshHandler getRefreshHandler() {
1516         return refreshHandler_;
1517     }
1518 
1519     /**
1520      * Sets the script pre processor for this webclient.
1521      * @param scriptPreProcessor the new preprocessor or null if none is specified
1522      */
1523     public void setScriptPreProcessor(final ScriptPreProcessor scriptPreProcessor) {
1524         scriptPreProcessor_ = scriptPreProcessor;
1525     }
1526 
1527     /**
1528      * Returns the script pre processor for this webclient.
1529      * @return the pre processor or null of one hasn't been set
1530      */
1531     public ScriptPreProcessor getScriptPreProcessor() {
1532         return scriptPreProcessor_;
1533     }
1534 
1535     /**
1536      * Sets the active X object map for this webclient. The <code>Map</code> is used to map the
1537      * string passed into the <code>ActiveXObject</code> constructor to a java class name. Therefore
1538      * you can emulate <code>ActiveXObject</code>s in a web page's JavaScript by mapping the object
1539      * name to a java class to emulate the active X object.
1540      * @param activeXObjectMap the new preprocessor or null if none is specified
1541      */
1542     public void setActiveXObjectMap(final Map<String, String> activeXObjectMap) {
1543         activeXObjectMap_ = activeXObjectMap;
1544     }
1545 
1546     /**
1547      * Returns the active X object map for this webclient.
1548      * @return the active X object map
1549      */
1550     public Map<String, String> getActiveXObjectMap() {
1551         return activeXObjectMap_;
1552     }
1553 
1554     /**
1555      * Returns the MSXML ActiveX object factory (if supported).
1556      * @return the msxmlActiveXObjectFactory
1557      */
1558     public MSXMLActiveXObjectFactory getMSXMLActiveXObjectFactory() {
1559         return msxmlActiveXObjectFactory_;
1560     }
1561 
1562     /**
1563      * Sets the listener for messages generated by the HTML parser.
1564      * @param listener the new listener, {@code null} if messages should be totally ignored
1565      */
1566     public void setHTMLParserListener(final HTMLParserListener listener) {
1567         htmlParserListener_ = listener;
1568     }
1569 
1570     /**
1571      * Gets the configured listener for messages generated by the HTML parser.
1572      * @return {@code null} if no listener is defined (default value)
1573      */
1574     public HTMLParserListener getHTMLParserListener() {
1575         return htmlParserListener_;
1576     }
1577 
1578     /**
1579      * Returns the CSS error handler used by this web client when CSS problems are encountered.
1580      * @return the CSS error handler used by this web client when CSS problems are encountered
1581      * @see DefaultCssErrorHandler
1582      * @see SilentCssErrorHandler
1583      */
1584     public ErrorHandler getCssErrorHandler() {
1585         return cssErrorHandler_;
1586     }
1587 
1588     /**
1589      * Sets the CSS error handler used by this web client when CSS problems are encountered.
1590      * @param cssErrorHandler the CSS error handler used by this web client when CSS problems are encountered
1591      * @see DefaultCssErrorHandler
1592      * @see SilentCssErrorHandler
1593      */
1594     public void setCssErrorHandler(final ErrorHandler cssErrorHandler) {
1595         WebAssert.notNull("cssErrorHandler", cssErrorHandler);
1596         cssErrorHandler_ = cssErrorHandler;
1597     }
1598 
1599     /**
1600      * Sets the number of milliseconds that a script is allowed to execute before being terminated.
1601      * A value of 0 or less means no timeout.
1602      *
1603      * @param timeout the timeout value, in milliseconds
1604      */
1605     public void setJavaScriptTimeout(final long timeout) {
1606         scriptEngine_.setJavaScriptTimeout(timeout);
1607     }
1608 
1609     /**
1610      * Returns the number of milliseconds that a script is allowed to execute before being terminated.
1611      * A value of 0 or less means no timeout.
1612      *
1613      * @return the timeout value, in milliseconds
1614      */
1615     public long getJavaScriptTimeout() {
1616         return scriptEngine_.getJavaScriptTimeout();
1617     }
1618 
1619     /**
1620      * Gets the current listener for encountered incorrectness (except HTML parsing messages that
1621      * are handled by the HTML parser listener). Default value is an instance of
1622      * {@link IncorrectnessListenerImpl}.
1623      * @return the current listener (not {@code null})
1624      */
1625     public IncorrectnessListener getIncorrectnessListener() {
1626         return incorrectnessListener_;
1627     }
1628 
1629     /**
1630      * Returns the current HTML incorrectness listener.
1631      * @param listener the new value (not {@code null})
1632      */
1633     public void setIncorrectnessListener(final IncorrectnessListener listener) {
1634         if (listener == null) {
1635             throw new NullPointerException("Null incorrectness listener.");
1636         }
1637         incorrectnessListener_ = listener;
1638     }
1639 
1640     /**
1641      * Returns the WebConsole.
1642      * @return the web console
1643      */
1644     public WebConsole getWebConsole() {
1645         if (webConsole_ == null) {
1646             webConsole_ = new WebConsole();
1647         }
1648         return webConsole_;
1649     }
1650 
1651     /**
1652      * Gets the current AJAX controller.
1653      * @return the controller
1654      */
1655     public AjaxController getAjaxController() {
1656         return ajaxController_;
1657     }
1658 
1659     /**
1660      * Sets the current AJAX controller.
1661      * @param newValue the controller
1662      */
1663     public void setAjaxController(final AjaxController newValue) {
1664         if (newValue == null) {
1665             throw new NullPointerException();
1666         }
1667         ajaxController_ = newValue;
1668     }
1669 
1670     /**
1671      * Sets the attachment handler.
1672      * @param handler the new attachment handler
1673      */
1674     public void setAttachmentHandler(final AttachmentHandler handler) {
1675         attachmentHandler_ = handler;
1676     }
1677 
1678     /**
1679      * Returns the current attachment handler.
1680      * @return the current attachment handler
1681      */
1682     public AttachmentHandler getAttachmentHandler() {
1683         return attachmentHandler_;
1684     }
1685 
1686     /**
1687      * Sets the applet confirm handler.
1688      * @param handler the new applet confirm handler handler
1689      */
1690     public void setAppletConfirmHandler(final AppletConfirmHandler handler) {
1691         appletConfirmHandler_ = handler;
1692     }
1693 
1694     /**
1695      * Returns the current applet confirm handler.
1696      * @return the current applet confirm handler
1697      */
1698     public AppletConfirmHandler getAppletConfirmHandler() {
1699         return appletConfirmHandler_;
1700     }
1701 
1702     /**
1703      * Sets the onbeforeunload handler for this webclient.
1704      * @param onbeforeunloadHandler the new onbeforeunloadHandler or null if none is specified
1705      */
1706     public void setOnbeforeunloadHandler(final OnbeforeunloadHandler onbeforeunloadHandler) {
1707         onbeforeunloadHandler_ = onbeforeunloadHandler;
1708     }
1709 
1710     /**
1711      * Returns the onbeforeunload handler for this webclient.
1712      * @return the onbeforeunload handler or null if one hasn't been set
1713      */
1714     public OnbeforeunloadHandler getOnbeforeunloadHandler() {
1715         return onbeforeunloadHandler_;
1716     }
1717 
1718     /**
1719      * Gets the cache currently being used.
1720      * @return the cache (may not be null)
1721      */
1722     public Cache getCache() {
1723         return cache_;
1724     }
1725 
1726     /**
1727      * Sets the cache to use.
1728      * @param cache the new cache (must not be {@code null})
1729      */
1730     public void setCache(final Cache cache) {
1731         if (cache == null) {
1732             throw new IllegalArgumentException("cache should not be null!");
1733         }
1734         cache_ = cache;
1735     }
1736 
1737     /**
1738      * Keeps track of the current window. Inspired by WebTest's logic to track the current response.
1739      */
1740     private static final class CurrentWindowTracker implements WebWindowListener, Serializable {
1741         private final WebClient webClient_;
1742 
1743         private CurrentWindowTracker(final WebClient webClient) {
1744             webClient_ = webClient;
1745         }
1746 
1747         /**
1748          * {@inheritDoc}
1749          */
1750         @Override
1751         public void webWindowClosed(final WebWindowEvent event) {
1752             final WebWindow window = event.getWebWindow();
1753             if (window instanceof TopLevelWindow) {
1754                 webClient_.topLevelWindows_.remove(window);
1755                 if (window == webClient_.getCurrentWindow()) {
1756                     if (webClient_.topLevelWindows_.isEmpty()) {
1757                         // Must always have at least window, and there are no top-level windows left; must create one.
1758                         final TopLevelWindow newWindow = new TopLevelWindow("", webClient_);
1759                         webClient_.topLevelWindows_.push(newWindow);
1760                         webClient_.setCurrentWindow(newWindow);
1761                     }
1762                     else {
1763                         // The current window is now the previous top-level window.
1764                         webClient_.setCurrentWindow(webClient_.topLevelWindows_.peek());
1765                     }
1766                 }
1767             }
1768             else if (window == webClient_.getCurrentWindow()) {
1769                 // The current window is now the last top-level window.
1770                 webClient_.setCurrentWindow(webClient_.topLevelWindows_.peek());
1771             }
1772         }
1773 
1774         /**
1775          * {@inheritDoc}
1776          */
1777         @Override
1778         public void webWindowContentChanged(final WebWindowEvent event) {
1779             final WebWindow window = event.getWebWindow();
1780             boolean use = false;
1781             if (window instanceof DialogWindow) {
1782                 use = true;
1783             }
1784             else if (window instanceof TopLevelWindow) {
1785                 use = event.getOldPage() == null;
1786             }
1787             else if (window instanceof FrameWindow) {
1788                 final FrameWindow fw = (FrameWindow) window;
1789                 final String enclosingPageState = fw.getEnclosingPage().getDocumentElement().getReadyState();
1790                 final URL frameUrl = fw.getEnclosedPage().getUrl();
1791                 if (!DomNode.READY_STATE_COMPLETE.equals(enclosingPageState) || frameUrl == URL_ABOUT_BLANK) {
1792                     return;
1793                 }
1794 
1795                 // now looks at the visibility of the frame window
1796                 final BaseFrameElement frameElement = fw.getFrameElement();
1797                 if (frameElement.isDisplayed()) {
1798                     final Object element = frameElement.getScriptableObject();
1799                     final HTMLElement htmlElement = (HTMLElement) element;
1800                     final ComputedCSSStyleDeclaration style =
1801                             htmlElement.getWindow().getComputedStyle(htmlElement, null);
1802                     use = style.getCalculatedWidth(false, false) != 0
1803                             && style.getCalculatedHeight(false, false) != 0;
1804                 }
1805             }
1806             if (use) {
1807                 webClient_.setCurrentWindow(window);
1808             }
1809         }
1810 
1811         /**
1812          * {@inheritDoc}
1813          */
1814         @Override
1815         public void webWindowOpened(final WebWindowEvent event) {
1816             final WebWindow window = event.getWebWindow();
1817             if (window instanceof TopLevelWindow) {
1818                 final TopLevelWindow tlw = (TopLevelWindow) window;
1819                 webClient_.topLevelWindows_.push(tlw);
1820             }
1821             // Page is not loaded yet, don't set it now as current window.
1822         }
1823     }
1824 
1825     /**
1826      * Closes all opened windows, stopping all background JavaScript processing.
1827      *
1828      * {@inheritDoc}
1829      */
1830     @Override
1831     public void close() {
1832         // NB: this implementation is too simple as a new TopLevelWindow may be opened by
1833         // some JS script while we are closing the others
1834         final List<TopLevelWindow> topWindows = new ArrayList<>(topLevelWindows_);
1835         for (final TopLevelWindow topWindow : topWindows) {
1836             if (topLevelWindows_.contains(topWindow)) {
1837                 try {
1838                     topWindow.close();
1839                 }
1840                 catch (final Exception e) {
1841                     LOG.error("Exception while closing a topLevelWindow", e);
1842                 }
1843             }
1844         }
1845 
1846         // do this after closing the windows, otherwise some unload event might
1847         // start a new window that will start the thread again
1848         if (scriptEngine_ != null) {
1849             try {
1850                 scriptEngine_.shutdown();
1851             }
1852             catch (final Exception e) {
1853                 LOG.error("Exception while shutdown the scriptEngine", e);
1854             }
1855         }
1856 
1857         try {
1858             webConnection_.close();
1859         }
1860         catch (final Exception e) {
1861             LOG.error("Exception while closing the connection", e);
1862         }
1863 
1864         cache_.clear();
1865     }
1866 
1867     /**
1868      * <p><span style="color:red">Experimental API: May be changed in next release
1869      * and may not yet work perfectly!</span></p>
1870      *
1871      * <p>This method blocks until all background JavaScript tasks have finished executing. Background
1872      * JavaScript tasks are JavaScript tasks scheduled for execution via <tt>window.setTimeout</tt>,
1873      * <tt>window.setInterval</tt> or asynchronous <tt>XMLHttpRequest</tt>.</p>
1874      *
1875      * <p>If a job is scheduled to begin executing after <tt>(now + timeoutMillis)</tt>, this method will
1876      * wait for <tt>timeoutMillis</tt> milliseconds and then return a value greater than <tt>0</tt>. This
1877      * method will never block longer than <tt>timeoutMillis</tt> milliseconds.</p>
1878      *
1879      * <p>Use this method instead of {@link #waitForBackgroundJavaScriptStartingBefore(long)} if you
1880      * don't know when your background JavaScript is supposed to start executing, but you're fairly sure
1881      * that you know how long it should take to finish executing.</p>
1882      *
1883      * @param timeoutMillis the maximum amount of time to wait (in milliseconds)
1884      * @return the number of background JavaScript jobs still executing or waiting to be executed when this
1885      *         method returns; will be <tt>0</tt> if there are no jobs left to execute
1886      */
1887     public int waitForBackgroundJavaScript(final long timeoutMillis) {
1888         int count = 0;
1889         final long endTime = System.currentTimeMillis() + timeoutMillis;
1890         for (Iterator<WeakReference<JavaScriptJobManager>> i = jobManagers_.iterator(); i.hasNext();) {
1891             final JavaScriptJobManager jobManager;
1892             final WeakReference<JavaScriptJobManager> reference;
1893             try {
1894                 reference = i.next();
1895                 jobManager = reference.get();
1896                 if (jobManager == null) {
1897                     i.remove();
1898                     continue;
1899                 }
1900             }
1901             catch (final ConcurrentModificationException e) {
1902                 i = jobManagers_.iterator();
1903                 count = 0;
1904                 continue;
1905             }
1906 
1907             final long newTimeout = endTime - System.currentTimeMillis();
1908             count += jobManager.waitForJobs(newTimeout);
1909         }
1910         if (count != getAggregateJobCount()) {
1911             final long newTimeout = endTime - System.currentTimeMillis();
1912             return waitForBackgroundJavaScript(newTimeout);
1913         }
1914         return count;
1915     }
1916 
1917     /**
1918      * <p><span style="color:red">Experimental API: May be changed in next release
1919      * and may not yet work perfectly!</span></p>
1920      *
1921      * <p>This method blocks until all background JavaScript tasks scheduled to start executing before
1922      * <tt>(now + delayMillis)</tt> have finished executing. Background JavaScript tasks are JavaScript
1923      * tasks scheduled for execution via <tt>window.setTimeout</tt>, <tt>window.setInterval</tt> or
1924      * asynchronous <tt>XMLHttpRequest</tt>.</p>
1925      *
1926      * <p>If there is no background JavaScript task currently executing, and there is no background JavaScript
1927      * task scheduled to start executing within the specified time, this method returns immediately -- even
1928      * if there are tasks scheduled to be executed after <tt>(now + delayMillis)</tt>.</p>
1929      *
1930      * <p>Note that the total time spent executing a background JavaScript task is never known ahead of
1931      * time, so this method makes no guarantees as to how long it will block.</p>
1932      *
1933      * <p>Use this method instead of {@link #waitForBackgroundJavaScript(long)} if you know roughly when
1934      * your background JavaScript is supposed to start executing, but you're not necessarily sure how long
1935      * it will take to execute.</p>
1936      *
1937      * @param delayMillis the delay which determines the background tasks to wait for (in milliseconds)
1938      * @return the number of background JavaScript jobs still executing or waiting to be executed when this
1939      *         method returns; will be <tt>0</tt> if there are no jobs left to execute
1940      */
1941     public int waitForBackgroundJavaScriptStartingBefore(final long delayMillis) {
1942         int count = 0;
1943         final long endTime = System.currentTimeMillis() + delayMillis;
1944         for (Iterator<WeakReference<JavaScriptJobManager>> i = jobManagers_.iterator(); i.hasNext();) {
1945             final JavaScriptJobManager jobManager;
1946             final WeakReference<JavaScriptJobManager> reference;
1947             try {
1948                 reference = i.next();
1949                 jobManager = reference.get();
1950                 if (jobManager == null) {
1951                     i.remove();
1952                     continue;
1953                 }
1954             }
1955             catch (final ConcurrentModificationException e) {
1956                 i = jobManagers_.iterator();
1957                 count = 0;
1958                 continue;
1959             }
1960             final long newDelay = endTime - System.currentTimeMillis();
1961             count += jobManager.waitForJobsStartingBefore(newDelay);
1962         }
1963         if (count != getAggregateJobCount()) {
1964             final long newDelay = endTime - System.currentTimeMillis();
1965             return waitForBackgroundJavaScriptStartingBefore(newDelay);
1966         }
1967         return count;
1968     }
1969 
1970     /**
1971      * Returns the aggregate background JavaScript job count across all windows.
1972      * @return the aggregate background JavaScript job count across all windows
1973      */
1974     private int getAggregateJobCount() {
1975         int count = 0;
1976         for (Iterator<WeakReference<JavaScriptJobManager>> i = jobManagers_.iterator(); i.hasNext();) {
1977             final JavaScriptJobManager jobManager;
1978             final WeakReference<JavaScriptJobManager> reference;
1979             try {
1980                 reference = i.next();
1981                 jobManager = reference.get();
1982                 if (jobManager == null) {
1983                     i.remove();
1984                     continue;
1985                 }
1986             }
1987             catch (final ConcurrentModificationException e) {
1988                 i = jobManagers_.iterator();
1989                 count = 0;
1990                 continue;
1991             }
1992             final int jobCount = jobManager.getJobCount();
1993             count += jobCount;
1994         }
1995         return count;
1996     }
1997 
1998     /**
1999      * When we deserialize, re-initializie transient fields.
2000      * @param in the object input stream
2001      * @throws IOException if an error occurs
2002      * @throws ClassNotFoundException if an error occurs
2003      */
2004     private void readObject(final ObjectInputStream in) throws IOException, ClassNotFoundException {
2005         in.defaultReadObject();
2006 
2007         webConnection_ = createWebConnection();
2008         scriptEngine_ = new JavaScriptEngine(this);
2009         jobManagers_ = Collections.synchronizedList(new ArrayList<WeakReference<JavaScriptJobManager>>());
2010 
2011         if (getBrowserVersion().hasFeature(JS_XML_SUPPORT_VIA_ACTIVEXOBJECT)) {
2012             initMSXMLActiveX();
2013         }
2014     }
2015 
2016     private WebConnection createWebConnection() {
2017         if (GAEUtils.isGaeMode()) {
2018             return new UrlFetchWebConnection(this);
2019         }
2020 
2021         return new HttpWebConnection(this);
2022     }
2023 
2024     private static class LoadJob {
2025         private final WebWindow requestingWindow_;
2026         private final String target_;
2027         private final WebResponse response_;
2028         private final URL urlWithOnlyHashChange_;
2029         private final WeakReference<Page> originalPage_;
2030         private final WebRequest request_;
2031 
2032         LoadJob(final WebRequest request, final WebWindow requestingWindow, final String target,
2033                 final WebResponse response) {
2034             request_ = request;
2035             requestingWindow_ = requestingWindow;
2036             target_ = target;
2037             response_ = response;
2038             urlWithOnlyHashChange_ = null;
2039             originalPage_ = new WeakReference<>(requestingWindow.getEnclosedPage());
2040         }
2041 
2042         LoadJob(final WebRequest request, final WebWindow requestingWindow, final String target,
2043                 final URL urlWithOnlyHashChange) {
2044             request_ = request;
2045             requestingWindow_ = requestingWindow;
2046             target_ = target;
2047             response_ = null;
2048             urlWithOnlyHashChange_ = urlWithOnlyHashChange;
2049             originalPage_ = new WeakReference<>(requestingWindow.getEnclosedPage());
2050         }
2051 
2052         public boolean isOutdated() {
2053             if (target_ != null && !target_.isEmpty()) {
2054                 return false;
2055             }
2056             else if (requestingWindow_.isClosed()) {
2057                 return true;
2058             }
2059             else if (requestingWindow_.getEnclosedPage() != originalPage_.get()) {
2060                 return true;
2061             }
2062 
2063             return false;
2064         }
2065     }
2066 
2067     private final List<LoadJob> loadQueue_ = new ArrayList<>();
2068 
2069     /**
2070      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2071      *
2072      * Perform the downloads and stores it for loading later into a window.
2073      * In the future downloads should be performed in parallel in separated threads.
2074      * TODO: refactor it before next release.
2075      * @param requestingWindow the window from which the request comes
2076      * @param target the name of the target window
2077      * @param request the request to perform
2078      * @param checkHash if true check for hashChenage
2079      * @param forceLoad if true always load the request even if there is already the same in the queue
2080      * @param description information about the origin of the request. Useful for debugging.
2081      */
2082     public void download(final WebWindow requestingWindow, final String target,
2083         final WebRequest request, final boolean checkHash, final boolean forceLoad, final String description) {
2084         final WebWindow win = resolveWindow(requestingWindow, target);
2085         final URL url = request.getUrl();
2086         boolean justHashJump = false;
2087 
2088         if (win != null && HttpMethod.POST != request.getHttpMethod()) {
2089             final Page page = win.getEnclosedPage();
2090             if (page != null) {
2091                 if (page.isHtmlPage() && !((HtmlPage) page).isOnbeforeunloadAccepted()) {
2092                     return;
2093                 }
2094 
2095                 if (checkHash) {
2096                     final URL current = page.getUrl();
2097                     justHashJump =
2098                             HttpMethod.GET == request.getHttpMethod()
2099                             && UrlUtils.sameFile(url, current)
2100                             && null != url.getRef();
2101                 }
2102             }
2103         }
2104 
2105         synchronized (loadQueue_) {
2106             // verify if this load job doesn't already exist
2107             for (final LoadJob loadJob : loadQueue_) {
2108                 if (loadJob.response_ == null) {
2109                     continue;
2110                 }
2111                 final WebRequest otherRequest = loadJob.request_;
2112                 final URL otherUrl = otherRequest.getUrl();
2113 
2114                 // TODO: investigate but it seems that IE considers query string too but not FF
2115                 if (!forceLoad
2116                     && url.getPath().equals(otherUrl.getPath()) // fail fast
2117                     && url.toString().equals(otherUrl.toString())
2118                     && request.getRequestParameters().equals(otherRequest.getRequestParameters())
2119                     && StringUtils.equals(request.getRequestBody(), otherRequest.getRequestBody())) {
2120                     return; // skip it;
2121                 }
2122             }
2123         }
2124 
2125         final LoadJob loadJob;
2126         if (justHashJump) {
2127             loadJob = new LoadJob(request, requestingWindow, target, url);
2128         }
2129         else {
2130             try {
2131                 final WebResponse response = loadWebResponse(request);
2132                 loadJob = new LoadJob(request, requestingWindow, target, response);
2133             }
2134             catch (final IOException e) {
2135                 throw new RuntimeException(e);
2136             }
2137         }
2138         synchronized (loadQueue_) {
2139             loadQueue_.add(loadJob);
2140         }
2141     }
2142 
2143     /**
2144      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2145      *
2146      * Loads downloaded responses into the corresponding windows.
2147      * TODO: refactor it before next release.
2148      * @throws IOException in case of exception
2149      * @throws FailingHttpStatusCodeException in case of exception
2150      */
2151     public void loadDownloadedResponses() throws FailingHttpStatusCodeException, IOException {
2152         final List<LoadJob> queue;
2153 
2154         // synchronize access to the loadQueue_,
2155         // to be sure no job is ignored
2156         synchronized (loadQueue_) {
2157             if (loadQueue_.isEmpty()) {
2158                 return;
2159             }
2160             queue = new ArrayList<>(loadQueue_);
2161             loadQueue_.clear();
2162         }
2163 
2164         final HashSet<WebWindow> updatedWindows = new HashSet<>();
2165         for (int i = queue.size() - 1; i >= 0; --i) {
2166             final LoadJob loadJob = queue.get(i);
2167             if (loadJob.isOutdated()) {
2168                 LOG.info("No usage of download: " + loadJob);
2169                 continue;
2170             }
2171 
2172             final WebWindow window = resolveWindow(loadJob.requestingWindow_, loadJob.target_);
2173             if (!updatedWindows.contains(window)) {
2174                 final WebWindow win = openTargetWindow(loadJob.requestingWindow_, loadJob.target_, "_self");
2175                 if (loadJob.urlWithOnlyHashChange_ != null) {
2176                     final HtmlPage page = (HtmlPage) loadJob.requestingWindow_.getEnclosedPage();
2177                     final String oldURL = page.getUrl().toExternalForm();
2178 
2179                     // update request url
2180                     final WebRequest req = page.getWebResponse().getWebRequest();
2181                     req.setUrl(loadJob.urlWithOnlyHashChange_);
2182 
2183                     // update location.hash
2184                     final Window jsWindow = (Window) win.getScriptableObject();
2185                     if (null != jsWindow) {
2186                         final Location location = jsWindow.getLocation();
2187                         location.setHash(oldURL, loadJob.urlWithOnlyHashChange_.getRef());
2188                     }
2189 
2190                     // add to history
2191                     win.getHistory().addPage(page);
2192                 }
2193                 else {
2194                     final Page pageBeforeLoad = win.getEnclosedPage();
2195                     loadWebResponseInto(loadJob.response_, win);
2196 
2197                     // start execution here.
2198                     if (scriptEngine_ != null) {
2199                         scriptEngine_.registerWindowAndMaybeStartEventLoop(win);
2200                     }
2201 
2202                     if (pageBeforeLoad != win.getEnclosedPage()) {
2203                         updatedWindows.add(win);
2204                     }
2205 
2206                     // check and report problems if needed
2207                     throwFailingHttpStatusCodeExceptionIfNecessary(loadJob.response_);
2208                 }
2209             }
2210             else {
2211                 LOG.info("No usage of download: " + loadJob);
2212             }
2213         }
2214     }
2215 
2216     /**
2217      * Returns the options object of this WebClient.
2218      * @return the options object
2219      */
2220     public WebClientOptions getOptions() {
2221         return options_;
2222     }
2223 
2224     /**
2225      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2226      *
2227      * Returns the internals object of this WebClient.
2228      * @return the internals object
2229      */
2230     public WebClientInternals getInternals() {
2231         return internals_;
2232     }
2233 
2234     /**
2235      * Gets the holder for the different storages.
2236      * <p><span style="color:red">Experimental API: May be changed in next release!</span></p>
2237      * @return the holder
2238      */
2239     public StorageHolder getStorageHolder() {
2240         return storageHolder_;
2241     }
2242 
2243     /**
2244      * Returns the currently configured cookies applicable to the specified URL, in an unmodifiable set.
2245      * If disabled, this returns an empty set.
2246      * @param url the URL on which to filter the returned cookies
2247      * @return the currently configured cookies applicable to the specified URL, in an unmodifiable set
2248      */
2249     public synchronized Set<Cookie> getCookies(final URL url) {
2250         final CookieManager cookieManager = getCookieManager();
2251 
2252         if (!cookieManager.isCookiesEnabled()) {
2253             return Collections.<Cookie>emptySet();
2254         }
2255 
2256         final URL normalizedUrl = cookieManager.replaceForCookieIfNecessary(url);
2257 
2258         final String host = normalizedUrl.getHost();
2259         // URLs like "about:blank" don't have cookies and we need to catch these
2260         // cases here before HttpClient complains
2261         if (host.isEmpty()) {
2262             return Collections.emptySet();
2263         }
2264 
2265         final String path = normalizedUrl.getPath();
2266         final String protocol = normalizedUrl.getProtocol();
2267         final boolean secure = "https".equals(protocol);
2268 
2269         final int port = cookieManager.getPort(normalizedUrl);
2270 
2271         // discard expired cookies
2272         cookieManager.clearExpired(new Date());
2273 
2274         final List<org.apache.http.cookie.Cookie> all = Cookie.toHttpClient(cookieManager.getCookies());
2275         final List<org.apache.http.cookie.Cookie> matches = new ArrayList<>();
2276 
2277         if (all.size() > 0) {
2278             final CookieOrigin cookieOrigin = new CookieOrigin(host, port, path, secure);
2279             final CookieSpec cookieSpec = new HtmlUnitBrowserCompatCookieSpec(getBrowserVersion());
2280             for (final org.apache.http.cookie.Cookie cookie : all) {
2281                 if (cookieSpec.match(cookie, cookieOrigin)) {
2282                     matches.add(cookie);
2283                 }
2284             }
2285         }
2286 
2287         final Set<Cookie> cookies = new LinkedHashSet<>();
2288         cookies.addAll(Cookie.fromHttpClient(matches));
2289         return Collections.unmodifiableSet(cookies);
2290     }
2291 
2292     /**
2293      * Parses the given cookie and adds this to our cookie store.
2294      * @param cookieString the string to parse
2295      * @param pageUrl the url of the page that likes to set the cookie
2296      * @param origin the requester
2297      */
2298     public void addCookie(final String cookieString, final URL pageUrl, final Object origin) {
2299         final BrowserVersion browserVersion = getBrowserVersion();
2300         final CookieManager cookieManager = getCookieManager();
2301         if (cookieManager.isCookiesEnabled()) {
2302             final CharArrayBuffer buffer = new CharArrayBuffer(cookieString.length() + 22);
2303             buffer.append("Set-Cookie: ");
2304             buffer.append(cookieString);
2305 
2306             final CookieSpec cookieSpec = new HtmlUnitBrowserCompatCookieSpec(browserVersion);
2307 
2308             try {
2309                 final List<org.apache.http.cookie.Cookie> cookies =
2310                         cookieSpec.parse(new BufferedHeader(buffer), cookieManager.buildCookieOrigin(pageUrl));
2311 
2312                 for (org.apache.http.cookie.Cookie cookie : cookies) {
2313                     final Cookie htmlUnitCookie = new Cookie((ClientCookie) cookie);
2314                     cookieManager.addCookie(htmlUnitCookie);
2315 
2316                     if (LOG.isDebugEnabled()) {
2317                         LOG.debug("Added cookie: '" + cookieString + "'");
2318                     }
2319                 }
2320             }
2321             catch (final MalformedCookieException e) {
2322                 getIncorrectnessListener().notify("set-cookie http-equiv meta tag: invalid cookie '"
2323                         + cookieString + "'; reason: '" + e.getMessage() + "'.", origin);
2324             }
2325         }
2326         else if (LOG.isDebugEnabled()) {
2327             LOG.debug("Skipped adding cookie: '" + cookieString + "'");
2328         }
2329     }
2330 }