View Javadoc
1   /*
2    * Copyright (c) 2002-2017 Gargoyle Software Inc.
3    *
4    * Licensed under the Apache License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    * http://www.apache.org/licenses/LICENSE-2.0
8    *
9    * Unless required by applicable law or agreed to in writing, software
10   * distributed under the License is distributed on an "AS IS" BASIS,
11   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12   * See the License for the specific language governing permissions and
13   * limitations under the License.
14   */
15  package com.gargoylesoftware.htmlunit;
16  
17  import static com.gargoylesoftware.htmlunit.BrowserVersionFeatures.DIALOGWINDOW_REFERER;
18  import static com.gargoylesoftware.htmlunit.BrowserVersionFeatures.HTTP_HEADER_UPGRADE_INSECURE_REQUEST;
19  import static com.gargoylesoftware.htmlunit.BrowserVersionFeatures.HTTP_REDIRECT_308;
20  import static com.gargoylesoftware.htmlunit.BrowserVersionFeatures.JS_XML_SUPPORT_VIA_ACTIVEXOBJECT;
21  import static com.gargoylesoftware.htmlunit.BrowserVersionFeatures.URL_MINIMAL_QUERY_ENCODING;
22  import static com.gargoylesoftware.htmlunit.BrowserVersionFeatures.WINDOW_EXECUTE_EVENTS;
23  import static java.nio.charset.StandardCharsets.ISO_8859_1;
24  import static java.nio.charset.StandardCharsets.UTF_8;
25  
26  import java.io.BufferedInputStream;
27  import java.io.File;
28  import java.io.FileInputStream;
29  import java.io.IOException;
30  import java.io.InputStream;
31  import java.io.ObjectInputStream;
32  import java.io.Serializable;
33  import java.lang.ref.WeakReference;
34  import java.net.MalformedURLException;
35  import java.net.URL;
36  import java.net.URLConnection;
37  import java.net.URLDecoder;
38  import java.nio.charset.Charset;
39  import java.util.ArrayList;
40  import java.util.Collections;
41  import java.util.ConcurrentModificationException;
42  import java.util.Date;
43  import java.util.HashMap;
44  import java.util.HashSet;
45  import java.util.Iterator;
46  import java.util.LinkedHashSet;
47  import java.util.List;
48  import java.util.Locale;
49  import java.util.Map;
50  import java.util.Set;
51  import java.util.Stack;
52  
53  import org.apache.commons.codec.DecoderException;
54  import org.apache.commons.lang3.StringUtils;
55  import org.apache.commons.logging.Log;
56  import org.apache.commons.logging.LogFactory;
57  import org.apache.http.HttpStatus;
58  import org.apache.http.NoHttpResponseException;
59  import org.apache.http.client.CredentialsProvider;
60  import org.apache.http.cookie.ClientCookie;
61  import org.apache.http.cookie.CookieOrigin;
62  import org.apache.http.cookie.CookieSpec;
63  import org.apache.http.cookie.MalformedCookieException;
64  import org.apache.http.message.BufferedHeader;
65  import org.apache.http.util.CharArrayBuffer;
66  import org.w3c.css.sac.ErrorHandler;
67  
68  import com.gargoylesoftware.htmlunit.activex.javascript.msxml.MSXMLActiveXObjectFactory;
69  import com.gargoylesoftware.htmlunit.attachment.Attachment;
70  import com.gargoylesoftware.htmlunit.attachment.AttachmentHandler;
71  import com.gargoylesoftware.htmlunit.gae.GAEUtils;
72  import com.gargoylesoftware.htmlunit.html.BaseFrameElement;
73  import com.gargoylesoftware.htmlunit.html.DomElement;
74  import com.gargoylesoftware.htmlunit.html.DomNode;
75  import com.gargoylesoftware.htmlunit.html.FrameWindow;
76  import com.gargoylesoftware.htmlunit.html.HTMLParserListener;
77  import com.gargoylesoftware.htmlunit.html.HtmlInlineFrame;
78  import com.gargoylesoftware.htmlunit.html.HtmlPage;
79  import com.gargoylesoftware.htmlunit.httpclient.HtmlUnitBrowserCompatCookieSpec;
80  import com.gargoylesoftware.htmlunit.javascript.AbstractJavaScriptEngine;
81  import com.gargoylesoftware.htmlunit.javascript.DefaultJavaScriptErrorListener;
82  import com.gargoylesoftware.htmlunit.javascript.JavaScriptEngine;
83  import com.gargoylesoftware.htmlunit.javascript.JavaScriptErrorListener;
84  import com.gargoylesoftware.htmlunit.javascript.background.JavaScriptJobManager;
85  import com.gargoylesoftware.htmlunit.javascript.host.Location;
86  import com.gargoylesoftware.htmlunit.javascript.host.Window;
87  import com.gargoylesoftware.htmlunit.javascript.host.css.ComputedCSSStyleDeclaration;
88  import com.gargoylesoftware.htmlunit.javascript.host.dom.Node;
89  import com.gargoylesoftware.htmlunit.javascript.host.event.Event;
90  import com.gargoylesoftware.htmlunit.javascript.host.html.HTMLDocument;
91  import com.gargoylesoftware.htmlunit.javascript.host.html.HTMLElement;
92  import com.gargoylesoftware.htmlunit.javascript.host.html.HTMLIFrameElement;
93  import com.gargoylesoftware.htmlunit.protocol.data.DataURLConnection;
94  import com.gargoylesoftware.htmlunit.util.Cookie;
95  import com.gargoylesoftware.htmlunit.util.NameValuePair;
96  import com.gargoylesoftware.htmlunit.util.UrlUtils;
97  
98  import net.sourceforge.htmlunit.corejs.javascript.ScriptableObject;
99  
100 /**
101  * The main starting point in HtmlUnit: this class simulates a web browser.
102  * <p>
103  * A standard usage of HtmlUnit will start with using the {@link #getPage(String)} method
104  * (or {@link #getPage(URL)}) to load a first {@link Page}
105  * and will continue with further processing on this page depending on its type.
106  * </p>
107  * <b>Example:</b><br>
108  * <br>
109  * <code>
110  * final WebClient webClient = new WebClient();<br>
111  * final {@link HtmlPage} startPage = webClient.getPage("http://htmlunit.sf.net");<br>
112  * assertEquals("HtmlUnit - Welcome to HtmlUnit", startPage.{@link HtmlPage#getTitleText() getTitleText}());
113  * </code>
114  * <p>
115  * Note: a {@link WebClient} instance is <b>not thread safe</b>. It is intended to be used from a single thread.
116  * </p>
117  * @author <a href="mailto:mbowler@GargoyleSoftware.com">Mike Bowler</a>
118  * @author <a href="mailto:gudujarlson@sf.net">Mike J. Bresnahan</a>
119  * @author Dominique Broeglin
120  * @author Noboru Sinohara
121  * @author <a href="mailto:chen_jun@users.sourceforge.net">Chen Jun</a>
122  * @author David K. Taylor
123  * @author <a href="mailto:cse@dynabean.de">Christian Sell</a>
124  * @author <a href="mailto:bcurren@esomnie.com">Ben Curren</a>
125  * @author Marc Guillemot
126  * @author Chris Erskine
127  * @author Daniel Gredler
128  * @author Sergey Gorelkin
129  * @author Hans Donner
130  * @author Paul King
131  * @author Ahmed Ashour
132  * @author Bruce Chapman
133  * @author Sudhan Moghe
134  * @author Martin Tamme
135  * @author Amit Manjhi
136  * @author Nicolas Belisle
137  * @author Ronald Brill
138  * @author Frank Danek
139  * @author Joerg Werner
140  */
141 public class WebClient implements Serializable, AutoCloseable {
142 
143     /** Logging support. */
144     private static final Log LOG = LogFactory.getLog(WebClient.class);
145 
146     /** Like the Firefox default value for {@code network.http.redirection-limit}. */
147     private static final int ALLOWED_REDIRECTIONS_SAME_URL = 20;
148 
149     private transient WebConnection webConnection_;
150     private CredentialsProvider credentialsProvider_ = new DefaultCredentialsProvider();
151     private CookieManager cookieManager_ = new CookieManager();
152     private transient AbstractJavaScriptEngine<?> scriptEngine_;
153     private final Map<String, String> requestHeaders_ = Collections.synchronizedMap(new HashMap<String, String>(89));
154     private IncorrectnessListener incorrectnessListener_ = new IncorrectnessListenerImpl();
155     private WebConsole webConsole_;
156 
157     private AlertHandler alertHandler_;
158     private ConfirmHandler confirmHandler_;
159     private PromptHandler promptHandler_;
160     private StatusHandler statusHandler_;
161     private AttachmentHandler attachmentHandler_;
162     private AppletConfirmHandler appletConfirmHandler_;
163     private AjaxController ajaxController_ = new AjaxController();
164 
165     private BrowserVersion browserVersion_;
166     private PageCreator pageCreator_ = new DefaultPageCreator();
167 
168     private final Set<WebWindowListener> webWindowListeners_ = new HashSet<>(5);
169     private final Stack<TopLevelWindow> topLevelWindows_ = new Stack<>(); // top-level windows
170     private final List<WebWindow> windows_ = Collections.synchronizedList(new ArrayList<WebWindow>()); // all windows
171     private transient List<WeakReference<JavaScriptJobManager>> jobManagers_ =
172             Collections.synchronizedList(new ArrayList<WeakReference<JavaScriptJobManager>>());
173     private WebWindow currentWindow_;
174 
175     private HTMLParserListener htmlParserListener_;
176     private ErrorHandler cssErrorHandler_ = new DefaultCssErrorHandler();
177     private OnbeforeunloadHandler onbeforeunloadHandler_;
178     private Cache cache_ = new Cache();
179 
180     /** target "_blank". */
181     private static final String TARGET_BLANK = "_blank";
182     /** target "_parent". */
183     private static final String TARGET_SELF = "_self";
184     /** target "_parent". */
185     private static final String TARGET_PARENT = "_parent";
186     /** target "_top". */
187     private static final String TARGET_TOP = "_top";
188 
189     /** "about:". */
190     public static final String ABOUT_SCHEME = "about:";
191     /** "about:blank". */
192     public static final String ABOUT_BLANK = ABOUT_SCHEME + "blank";
193     /** URL for "about:blank". */
194     public static final URL URL_ABOUT_BLANK = UrlUtils.toUrlSafe(ABOUT_BLANK);
195 
196     private ScriptPreProcessor scriptPreProcessor_;
197 
198     private Map<String, String> activeXObjectMap_ = Collections.emptyMap();
199     private transient MSXMLActiveXObjectFactory msxmlActiveXObjectFactory_;
200     private RefreshHandler refreshHandler_ = new NiceRefreshHandler(2);
201     private JavaScriptErrorListener javaScriptErrorListener_ = new DefaultJavaScriptErrorListener();
202 
203     private WebClientOptions options_ = new WebClientOptions();
204     private WebClientInternals internals_ = new WebClientInternals(this);
205     private final StorageHolder storageHolder_ = new StorageHolder();
206 
207     private static final WebResponseData responseDataNoHttpResponse_ = new WebResponseData(
208         0, "No HTTP Response", Collections.<NameValuePair>emptyList());
209 
210     /**
211      * Creates a web client instance using the browser version returned by
212      * {@link BrowserVersion#getDefault()}.
213      */
214     public WebClient() {
215         this(BrowserVersion.getDefault());
216     }
217 
218     /**
219      * Creates a web client instance using the specified {@link BrowserVersion}.
220      * @param browserVersion the browser version to simulate
221      */
222     public WebClient(final BrowserVersion browserVersion) {
223         WebAssert.notNull("browserVersion", browserVersion);
224         init(browserVersion, new ProxyConfig());
225     }
226 
227     /**
228      * Creates an instance that will use the specified {@link BrowserVersion} and proxy server.
229      * @param browserVersion the browser version to simulate
230      * @param proxyHost the server that will act as proxy
231      * @param proxyPort the port to use on the proxy server
232      */
233     public WebClient(final BrowserVersion browserVersion, final String proxyHost, final int proxyPort) {
234         WebAssert.notNull("browserVersion", browserVersion);
235         WebAssert.notNull("proxyHost", proxyHost);
236         init(browserVersion, new ProxyConfig(proxyHost, proxyPort));
237     }
238 
239     /**
240      * Generic initialization logic used by all constructors. This method does not perform any
241      * parameter validation; such validation must be handled by the constructors themselves.
242      * @param browserVersion the browser version to simulate
243      * @param proxyConfig the proxy configuration to use
244      */
245     private void init(final BrowserVersion browserVersion, final ProxyConfig proxyConfig) {
246         browserVersion_ = browserVersion;
247         getOptions().setProxyConfig(proxyConfig);
248 
249         webConnection_ = createWebConnection(); // this has to be done after the browser version was set
250         scriptEngine_ = new JavaScriptEngine(this);
251         // The window must be constructed AFTER the script engine.
252         addWebWindowListener(new CurrentWindowTracker(this));
253         currentWindow_ = new TopLevelWindow("", this);
254         fireWindowOpened(new WebWindowEvent(currentWindow_, WebWindowEvent.OPEN, null, null));
255 
256         if (getBrowserVersion().hasFeature(JS_XML_SUPPORT_VIA_ACTIVEXOBJECT)) {
257             initMSXMLActiveX();
258         }
259     }
260 
261     private void initMSXMLActiveX() {
262         msxmlActiveXObjectFactory_ = new MSXMLActiveXObjectFactory();
263         // TODO [IE] initialize in #init or in #initialize?
264         try {
265             msxmlActiveXObjectFactory_.init(getBrowserVersion());
266         }
267         catch (final Exception e) {
268             LOG.error("Exception while initializing MSXML ActiveX for the page", e);
269             throw new ScriptException(null, e); // BUG: null is not useful.
270         }
271     }
272 
273     /**
274      * Returns the object that will resolve all URL requests.
275      *
276      * @return the connection that will be used
277      */
278     public WebConnection getWebConnection() {
279         return webConnection_;
280     }
281 
282     /**
283      * Sets the object that will resolve all URL requests.
284      *
285      * @param webConnection the new web connection
286      */
287     public void setWebConnection(final WebConnection webConnection) {
288         WebAssert.notNull("webConnection", webConnection);
289         webConnection_ = webConnection;
290     }
291 
292     /**
293      * Send a request to a server and return a Page that represents the
294      * response from the server. This page will be used to populate the provided window.
295      * <p>
296      * The returned {@link Page} will be created by the {@link PageCreator}
297      * configured by {@link #setPageCreator(PageCreator)}, if any.
298      * <p>
299      * The {@link DefaultPageCreator} will create a {@link Page} depending on the content type of the HTTP response,
300      * basically {@link HtmlPage} for HTML content, {@link com.gargoylesoftware.htmlunit.xml.XmlPage} for XML content,
301      * {@link TextPage} for other text content and {@link UnexpectedPage} for anything else.
302      *
303      * @param webWindow the WebWindow to load the result of the request into
304      * @param webRequest the web request
305      * @param <P> the page type
306      * @return the page returned by the server when the specified request was made in the specified window
307      * @throws IOException if an IO error occurs
308      * @throws FailingHttpStatusCodeException if the server returns a failing status code AND the property
309      *         {@link WebClientOptions#setThrowExceptionOnFailingStatusCode(boolean)} is set to true
310      *
311      * @see WebRequest
312      */
313     public <P extends Page> P getPage(final WebWindow webWindow, final WebRequest webRequest)
314             throws IOException, FailingHttpStatusCodeException {
315         return getPage(webWindow, webRequest, true);
316     }
317 
318     /**
319      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
320      *
321      * Send a request to a server and return a Page that represents the
322      * response from the server. This page will be used to populate the provided window.
323      * <p>
324      * The returned {@link Page} will be created by the {@link PageCreator}
325      * configured by {@link #setPageCreator(PageCreator)}, if any.
326      * <p>
327      * The {@link DefaultPageCreator} will create a {@link Page} depending on the content type of the HTTP response,
328      * basically {@link HtmlPage} for HTML content, {@link com.gargoylesoftware.htmlunit.xml.XmlPage} for XML content,
329      * {@link TextPage} for other text content and {@link UnexpectedPage} for anything else.
330      *
331      * @param webWindow the WebWindow to load the result of the request into
332      * @param webRequest the web request
333      * @param addToHistory true if the page should be part of the history
334      * @param <P> the page type
335      * @return the page returned by the server when the specified request was made in the specified window
336      * @throws IOException if an IO error occurs
337      * @throws FailingHttpStatusCodeException if the server returns a failing status code AND the property
338      *         {@link WebClientOptions#setThrowExceptionOnFailingStatusCode(boolean)} is set to true
339      *
340      * @see WebRequest
341      */
342     @SuppressWarnings("unchecked")
343     <P extends Page> P getPage(final WebWindow webWindow, final WebRequest webRequest,
344             final boolean addToHistory)
345         throws IOException, FailingHttpStatusCodeException {
346 
347         final Page page = webWindow.getEnclosedPage();
348 
349         if (page != null) {
350             final URL prev = page.getUrl();
351             final URL current = webRequest.getUrl();
352             if (UrlUtils.sameFile(current, prev)
353                         && current.getRef() != null
354                         && !StringUtils.equals(current.getRef(), prev.getRef())) {
355                 // We're just navigating to an anchor within the current page.
356                 page.getWebResponse().getWebRequest().setUrl(current);
357                 if (addToHistory) {
358                     webWindow.getHistory().addPage(page);
359                 }
360 
361                 final Window window = (Window) webWindow.getScriptableObject();
362                 if (window != null) { // js enabled
363                     window.getLocation().setHash(current.getRef());
364                     window.clearComputedStyles();
365                 }
366                 return (P) page;
367             }
368 
369             if (page.isHtmlPage()) {
370                 final HtmlPage htmlPage = (HtmlPage) page;
371                 if (!htmlPage.isOnbeforeunloadAccepted()) {
372                     if (LOG.isDebugEnabled()) {
373                         LOG.debug("The registered OnbeforeunloadHandler rejected to load a new page.");
374                     }
375                     return (P) page;
376                 }
377             }
378         }
379 
380         if (LOG.isDebugEnabled()) {
381             LOG.debug("Get page for window named '" + webWindow.getName() + "', using " + webRequest);
382         }
383 
384         final WebResponse webResponse;
385         final String protocol = webRequest.getUrl().getProtocol();
386         if ("javascript".equals(protocol)) {
387             webResponse = makeWebResponseForJavaScriptUrl(webWindow, webRequest.getUrl(), webRequest.getCharset());
388             if (webWindow.getEnclosedPage() != null && webWindow.getEnclosedPage().getWebResponse() == webResponse) {
389                 // a javascript:... url with result of type undefined didn't changed the page
390                 return (P) webWindow.getEnclosedPage();
391             }
392         }
393         else {
394             webResponse = loadWebResponse(webRequest);
395         }
396 
397         printContentIfNecessary(webResponse);
398         loadWebResponseInto(webResponse, webWindow);
399 
400         // start execution here
401         // note: we have to do this also if the server reports an error!
402         //       e.g. if the server returns a 404 error page that includes javascript
403         if (scriptEngine_ != null) {
404             scriptEngine_.registerWindowAndMaybeStartEventLoop(webWindow);
405         }
406 
407         // check and report problems if needed
408         throwFailingHttpStatusCodeExceptionIfNecessary(webResponse);
409         return (P) webWindow.getEnclosedPage();
410     }
411 
412     /**
413      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
414      *
415      * <p>Open a new web window and populate it with a page loaded by
416      * {@link #getPage(WebWindow,WebRequest)}</p>
417      *
418      * @param opener the web window that initiated the request
419      * @param target the name of the window to be opened (the name that will be passed into the
420      *        JavaScript <tt>open()</tt> method)
421      * @param params any parameters
422      * @param <P> the page type
423      * @return the new page
424      * @throws FailingHttpStatusCodeException if the server returns a failing status code AND the property
425      *         {@link WebClientOptions#setThrowExceptionOnFailingStatusCode(boolean)} is set to true.
426      * @throws IOException if an IO problem occurs
427      */
428     @SuppressWarnings("unchecked")
429     public <P extends Page> P getPage(final WebWindow opener, final String target, final WebRequest params)
430         throws FailingHttpStatusCodeException, IOException {
431         return (P) getPage(openTargetWindow(opener, target, TARGET_SELF), params);
432     }
433 
434     /**
435      * Convenient method to build a URL and load it into the current WebWindow as it would be done
436      * by {@link #getPage(WebWindow, WebRequest)}.
437      * @param url the URL of the new content
438      * @param <P> the page type
439      * @return the new page
440      * @throws FailingHttpStatusCodeException if the server returns a failing status code AND the property
441      *         {@link WebClientOptions#setThrowExceptionOnFailingStatusCode(boolean)} is set to true.
442      * @throws IOException if an IO problem occurs
443      * @throws MalformedURLException if no URL can be created from the provided string
444      */
445     @SuppressWarnings("unchecked")
446     public <P extends Page> P getPage(final String url) throws IOException, FailingHttpStatusCodeException,
447         MalformedURLException {
448         return (P) getPage(UrlUtils.toUrlUnsafe(url));
449     }
450 
451     /**
452      * Convenient method to load a URL into the current top WebWindow as it would be done
453      * by {@link #getPage(WebWindow, WebRequest)}.
454      * @param url the URL of the new content
455      * @param <P> the page type
456      * @return the new page
457      * @throws FailingHttpStatusCodeException if the server returns a failing status code AND the property
458      *         {@link WebClientOptions#setThrowExceptionOnFailingStatusCode(boolean)} is set to true.
459      * @throws IOException if an IO problem occurs
460      */
461     @SuppressWarnings("unchecked")
462     public <P extends Page> P getPage(final URL url) throws IOException, FailingHttpStatusCodeException {
463         return (P) getPage(getCurrentWindow().getTopWindow(),
464                 new WebRequest(url, getBrowserVersion().getHtmlAcceptHeader()));
465     }
466 
467     /**
468      * Convenient method to load a web request into the current top WebWindow.
469      * @param request the request parameters
470      * @param <P> the page type
471      * @return the new page
472      * @throws FailingHttpStatusCodeException if the server returns a failing status code AND the property
473      *         {@link WebClientOptions#setThrowExceptionOnFailingStatusCode(boolean)} is set to true.
474      * @throws IOException if an IO problem occurs
475      * @see #getPage(WebWindow,WebRequest)
476      */
477     @SuppressWarnings("unchecked")
478     public <P extends Page> P getPage(final WebRequest request) throws IOException,
479         FailingHttpStatusCodeException {
480         return (P) getPage(getCurrentWindow().getTopWindow(), request);
481     }
482 
483     /**
484      * <p>Creates a page based on the specified response and inserts it into the specified window. All page
485      * initialization and event notification is handled here.</p>
486      *
487      * <p>Note that if the page created is an attachment page, and an {@link AttachmentHandler} has been
488      * registered with this client, the page is <b>not</b> loaded into the specified window; in this case,
489      * the page is loaded into a new window, and attachment handling is delegated to the registered
490      * <tt>AttachmentHandler</tt>.</p>
491      *
492      * @param webResponse the response that will be used to create the new page
493      * @param webWindow the window that the new page will be placed within
494      * @throws IOException if an IO error occurs
495      * @throws FailingHttpStatusCodeException if the server returns a failing status code AND the property
496      *         {@link WebClientOptions#setThrowExceptionOnFailingStatusCode(boolean)} is set to true
497      * @return the newly created page
498      * @see #setAttachmentHandler(AttachmentHandler)
499      */
500     public Page loadWebResponseInto(final WebResponse webResponse, final WebWindow webWindow)
501         throws IOException, FailingHttpStatusCodeException {
502 
503         WebAssert.notNull("webResponse", webResponse);
504         WebAssert.notNull("webWindow", webWindow);
505 
506         if (webResponse.getStatusCode() == HttpStatus.SC_NO_CONTENT) {
507             return webWindow.getEnclosedPage();
508         }
509 
510         if (attachmentHandler_ != null && Attachment.isAttachment(webResponse)) {
511             final WebWindow w = openWindow(null, null, webWindow);
512             final Page page = pageCreator_.createPage(webResponse, w);
513             attachmentHandler_.handleAttachment(page);
514             return page;
515         }
516 
517         final Page oldPage = webWindow.getEnclosedPage();
518         if (oldPage != null) {
519             // Remove the old windows before create new ones.
520             oldPage.cleanUp();
521         }
522         Page newPage = null;
523         if (windows_.contains(webWindow) || getBrowserVersion().hasFeature(WINDOW_EXECUTE_EVENTS)) {
524             newPage = pageCreator_.createPage(webResponse, webWindow);
525 
526             if (windows_.contains(webWindow)) {
527                 fireWindowContentChanged(new WebWindowEvent(webWindow, WebWindowEvent.CHANGE, oldPage, newPage));
528 
529                 // The page being loaded may already have been replaced by another page via JavaScript code.
530                 if (webWindow.getEnclosedPage() == newPage) {
531                     newPage.initialize();
532                     // hack: onload should be fired the same way for all type of pages
533                     // here is a hack to handle non HTML pages
534                     if (webWindow instanceof FrameWindow && !newPage.isHtmlPage()) {
535                         final FrameWindow fw = (FrameWindow) webWindow;
536                         final BaseFrameElement frame = fw.getFrameElement();
537                         if (frame.hasEventHandlers("onload")) {
538                             if (LOG.isDebugEnabled()) {
539                                 LOG.debug("Executing onload handler for " + frame);
540                             }
541                             final Event event = new Event(frame, Event.TYPE_LOAD);
542                             ((Node) frame.getScriptableObject()).executeEventLocally(event);
543                         }
544                     }
545                 }
546             }
547         }
548         return newPage;
549     }
550 
551     /**
552      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span>
553      *
554      * <p>Logs the response's content if its status code indicates a request failure and
555      * {@link WebClientOptions#isPrintContentOnFailingStatusCode()} returns {@code true}.
556      *
557      * @param webResponse the response whose content may be logged
558      */
559     public void printContentIfNecessary(final WebResponse webResponse) {
560         if (getOptions().isPrintContentOnFailingStatusCode()) {
561             final int statusCode = webResponse.getStatusCode();
562             final boolean successful = statusCode >= HttpStatus.SC_OK && statusCode < HttpStatus.SC_MULTIPLE_CHOICES;
563             if (!successful) {
564                 final String contentType = webResponse.getContentType();
565                 LOG.info("statusCode=[" + statusCode + "] contentType=[" + contentType + "]");
566                 LOG.info(webResponse.getContentAsString());
567             }
568         }
569     }
570 
571     /**
572      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span>
573      *
574      * <p>Throws a {@link FailingHttpStatusCodeException} if the request's status code indicates a request
575      * failure and {@link WebClientOptions#isThrowExceptionOnFailingStatusCode()} returns {@code true}.
576      *
577      * @param webResponse the response which may trigger a {@link FailingHttpStatusCodeException}
578      */
579     public void throwFailingHttpStatusCodeExceptionIfNecessary(final WebResponse webResponse) {
580         final int statusCode = webResponse.getStatusCode();
581         final boolean successful = (statusCode >= HttpStatus.SC_OK && statusCode < HttpStatus.SC_MULTIPLE_CHOICES)
582             || statusCode == HttpStatus.SC_USE_PROXY
583             || statusCode == HttpStatus.SC_NOT_MODIFIED;
584         if (getOptions().isThrowExceptionOnFailingStatusCode() && !successful) {
585             throw new FailingHttpStatusCodeException(webResponse);
586         }
587     }
588 
589     /**
590      * Adds a header which will be sent with EVERY request from this client.
591      * @param name the name of the header to add
592      * @param value the value of the header to add
593      * @see #removeRequestHeader(String)
594      */
595     public void addRequestHeader(final String name, final String value) {
596         if ("cookie".equalsIgnoreCase(name)) {
597             throw new IllegalArgumentException("Do not add 'Cookie' header, use .getCookieManager() instead");
598         }
599         requestHeaders_.put(name, value);
600     }
601 
602     /**
603      * Removes a header from being sent with EVERY request from this client.
604      * @param name the name of the header to remove
605      * @see #addRequestHeader
606      */
607     public void removeRequestHeader(final String name) {
608         requestHeaders_.remove(name);
609     }
610 
611     /**
612      * Sets the credentials provider that will provide authentication information when
613      * trying to access protected information on a web server. This information is
614      * required when the server is using Basic HTTP authentication, NTLM authentication,
615      * or Digest authentication.
616      * @param credentialsProvider the new credentials provider to use to authenticate
617      */
618     public void setCredentialsProvider(final CredentialsProvider credentialsProvider) {
619         WebAssert.notNull("credentialsProvider", credentialsProvider);
620         credentialsProvider_ = credentialsProvider;
621     }
622 
623     /**
624      * Returns the credentials provider for this client instance. By default, this
625      * method returns an instance of {@link DefaultCredentialsProvider}.
626      * @return the credentials provider for this client instance
627      */
628     public CredentialsProvider getCredentialsProvider() {
629         return credentialsProvider_;
630     }
631 
632     /**
633      * This method is intended for testing only - use at your own risk.
634      * @return the current JavaScript engine (never {@code null})
635      */
636     public AbstractJavaScriptEngine<?> getJavaScriptEngine() {
637         return scriptEngine_;
638     }
639 
640     /**
641      * This method is intended for testing only - use at your own risk.
642      *
643      * @param engine the new script engine to use
644      */
645     public void setJavaScriptEngine(final AbstractJavaScriptEngine<?> engine) {
646         if (engine == null) {
647             throw new IllegalArgumentException("Can't set JavaScriptEngine to null");
648         }
649         scriptEngine_ = engine;
650     }
651 
652     /**
653      * Returns the cookie manager used by this web client.
654      * @return the cookie manager used by this web client
655      */
656     public CookieManager getCookieManager() {
657         return cookieManager_;
658     }
659 
660     /**
661      * Sets the cookie manager used by this web client.
662      * @param cookieManager the cookie manager used by this web client
663      */
664     public void setCookieManager(final CookieManager cookieManager) {
665         WebAssert.notNull("cookieManager", cookieManager);
666         cookieManager_ = cookieManager;
667     }
668 
669     /**
670      * Sets the alert handler for this webclient.
671      * @param alertHandler the new alerthandler or null if none is specified
672      */
673     public void setAlertHandler(final AlertHandler alertHandler) {
674         alertHandler_ = alertHandler;
675     }
676 
677     /**
678      * Returns the alert handler for this webclient.
679      * @return the alert handler or null if one hasn't been set
680      */
681     public AlertHandler getAlertHandler() {
682         return alertHandler_;
683     }
684 
685     /**
686      * Sets the handler that will be executed when the JavaScript method Window.confirm() is called.
687      * @param handler the new handler or null if no handler is to be used
688      */
689     public void setConfirmHandler(final ConfirmHandler handler) {
690         confirmHandler_ = handler;
691     }
692 
693     /**
694      * Returns the confirm handler.
695      * @return the confirm handler or null if one hasn't been set
696      */
697     public ConfirmHandler getConfirmHandler() {
698         return confirmHandler_;
699     }
700 
701     /**
702      * Sets the handler that will be executed when the JavaScript method Window.prompt() is called.
703      * @param handler the new handler or null if no handler is to be used
704      */
705     public void setPromptHandler(final PromptHandler handler) {
706         promptHandler_ = handler;
707     }
708 
709     /**
710      * Returns the prompt handler.
711      * @return the prompt handler or null if one hasn't been set
712      */
713     public PromptHandler getPromptHandler() {
714         return promptHandler_;
715     }
716 
717     /**
718      * Sets the status handler for this webclient.
719      * @param statusHandler the new status handler or null if none is specified
720      */
721     public void setStatusHandler(final StatusHandler statusHandler) {
722         statusHandler_ = statusHandler;
723     }
724 
725     /**
726      * Returns the status handler for this webclient.
727      * @return the status handler or null if one hasn't been set
728      */
729     public StatusHandler getStatusHandler() {
730         return statusHandler_;
731     }
732 
733     /**
734      * Sets the javascript error listener for this webclient.
735      * When setting to null, the {@link DefaultJavaScriptErrorListener} is used.
736      * @param javaScriptErrorListener the new JavaScriptErrorListener or null if none is specified
737      */
738     public void setJavaScriptErrorListener(final JavaScriptErrorListener javaScriptErrorListener) {
739         if (javaScriptErrorListener == null) {
740             javaScriptErrorListener_ = new DefaultJavaScriptErrorListener();
741         }
742         else {
743             javaScriptErrorListener_ = javaScriptErrorListener;
744         }
745     }
746 
747     /**
748      * Returns the javascript error listener for this webclient.
749      * @return the javascript error listener or null if one hasn't been set
750      */
751     public JavaScriptErrorListener getJavaScriptErrorListener() {
752         return javaScriptErrorListener_;
753     }
754 
755     /**
756      * Returns the current browser version.
757      * @return the current browser version
758      */
759     public BrowserVersion getBrowserVersion() {
760         return browserVersion_;
761     }
762 
763     /**
764      * Returns the "current" window for this client. This window (or its top window) will be used
765      * when <tt>getPage(...)</tt> is called without specifying a window.
766      * @return the "current" window for this client
767      */
768     public WebWindow getCurrentWindow() {
769         return currentWindow_;
770     }
771 
772     /**
773      * Sets the "current" window for this client. This is the window that will be used when
774      * <tt>getPage(...)</tt> is called without specifying a window.
775      * @param window the new "current" window for this client
776      */
777     public void setCurrentWindow(final WebWindow window) {
778         WebAssert.notNull("window", window);
779         if (currentWindow_ == window) {
780             return;
781         }
782         // onBlur event is triggered for focused element of old current window
783         if (currentWindow_ != null && !currentWindow_.isClosed()) {
784             final Page enclosedPage = currentWindow_.getEnclosedPage();
785             if (enclosedPage != null && enclosedPage.isHtmlPage()) {
786                 final DomElement focusedElement = ((HtmlPage) enclosedPage).getFocusedElement();
787                 if (focusedElement != null) {
788                     focusedElement.fireEvent(Event.TYPE_BLUR);
789                 }
790             }
791         }
792         currentWindow_ = window;
793 
794         // when marking an iframe window as current we have no need to move the focus
795         final boolean isIFrame = currentWindow_ instanceof FrameWindow
796                 && ((FrameWindow) currentWindow_).getFrameElement() instanceof HtmlInlineFrame;
797         if (!isIFrame) {
798             //1. activeElement becomes focused element for new current window
799             //2. onFocus event is triggered for focusedElement of new current window
800             final Page enclosedPage = currentWindow_.getEnclosedPage();
801             if (enclosedPage != null && enclosedPage.isHtmlPage()) {
802                 final Object jsWindow = currentWindow_.getScriptableObject();
803                 if (jsWindow instanceof Window) {
804                     final HTMLElement activeElement =
805                             ((HTMLDocument) ((Window) jsWindow).getDocument()).getActiveElement();
806                     if (activeElement != null) {
807                         ((HtmlPage) enclosedPage).setFocusedElement(activeElement.getDomNodeOrDie(), true);
808                     }
809                 }
810             }
811         }
812     }
813 
814     /**
815      * Adds a listener for {@link WebWindowEvent}s. All events from all windows associated with this
816      * client will be sent to the specified listener.
817      * @param listener a listener
818      */
819     public void addWebWindowListener(final WebWindowListener listener) {
820         WebAssert.notNull("listener", listener);
821         webWindowListeners_.add(listener);
822     }
823 
824     /**
825      * Removes a listener for {@link WebWindowEvent}s.
826      * @param listener a listener
827      */
828     public void removeWebWindowListener(final WebWindowListener listener) {
829         WebAssert.notNull("listener", listener);
830         webWindowListeners_.remove(listener);
831     }
832 
833     private void fireWindowContentChanged(final WebWindowEvent event) {
834         for (final WebWindowListener listener : new ArrayList<>(webWindowListeners_)) {
835             listener.webWindowContentChanged(event);
836         }
837     }
838 
839     private void fireWindowOpened(final WebWindowEvent event) {
840         for (final WebWindowListener listener : new ArrayList<>(webWindowListeners_)) {
841             listener.webWindowOpened(event);
842         }
843     }
844 
845     private void fireWindowClosed(final WebWindowEvent event) {
846         for (final WebWindowListener listener : new ArrayList<>(webWindowListeners_)) {
847             listener.webWindowClosed(event);
848         }
849     }
850 
851     /**
852      * Open a new window with the specified name. If the URL is non-null then attempt to load
853      * a page from that location and put it in the new window.
854      *
855      * @param url the URL to load content from or null if no content is to be loaded
856      * @param windowName the name of the new window
857      * @return the new window
858      */
859     public WebWindow openWindow(final URL url, final String windowName) {
860         WebAssert.notNull("windowName", windowName);
861         return openWindow(url, windowName, getCurrentWindow());
862     }
863 
864     /**
865      * Open a new window with the specified name. If the URL is non-null then attempt to load
866      * a page from that location and put it in the new window.
867      *
868      * @param url the URL to load content from or null if no content is to be loaded
869      * @param windowName the name of the new window
870      * @param opener the web window that is calling openWindow
871      * @return the new window
872      */
873     public WebWindow openWindow(final URL url, final String windowName, final WebWindow opener) {
874         final WebWindow window = openTargetWindow(opener, windowName, TARGET_BLANK);
875         final HtmlPage openerPage = (HtmlPage) opener.getEnclosedPage();
876         if (url != null) {
877             try {
878                 final WebRequest request = new WebRequest(url, getBrowserVersion().getHtmlAcceptHeader());
879                 if (getBrowserVersion().hasFeature(DIALOGWINDOW_REFERER)
880                         && openerPage != null) {
881                     final String referer = openerPage.getUrl().toExternalForm();
882                     request.setAdditionalHeader("Referer", referer);
883                 }
884                 getPage(window, request);
885             }
886             catch (final IOException e) {
887                 LOG.error("Error loading content into window", e);
888             }
889         }
890         else {
891             initializeEmptyWindow(window);
892         }
893         return window;
894     }
895 
896     /**
897      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
898      *
899      * Open the window with the specified name. The name may be a special
900      * target name of _self, _parent, _top, or _blank. An empty or null
901      * name is set to the default. The special target names are relative to
902      * the opener window.
903      *
904      * @param opener the web window that is calling openWindow
905      * @param windowName the name of the new window
906      * @param defaultName the default target if no name is given
907      * @return the new window
908      */
909     public WebWindow openTargetWindow(
910             final WebWindow opener, final String windowName, final String defaultName) {
911 
912         WebAssert.notNull("opener", opener);
913         WebAssert.notNull("defaultName", defaultName);
914 
915         String windowToOpen = windowName;
916         if (windowToOpen == null || windowToOpen.isEmpty()) {
917             windowToOpen = defaultName;
918         }
919 
920         WebWindow webWindow = resolveWindow(opener, windowToOpen);
921 
922         if (webWindow == null) {
923             if (TARGET_BLANK.equals(windowToOpen)) {
924                 windowToOpen = "";
925             }
926             webWindow = new TopLevelWindow(windowToOpen, this);
927             fireWindowOpened(new WebWindowEvent(webWindow, WebWindowEvent.OPEN, null, null));
928         }
929 
930         if (webWindow instanceof TopLevelWindow && webWindow != opener.getTopWindow()) {
931             ((TopLevelWindow) webWindow).setOpener(opener);
932         }
933 
934         return webWindow;
935     }
936 
937     private WebWindow resolveWindow(final WebWindow opener, final String name) {
938         if (name == null || name.isEmpty() || TARGET_SELF.equals(name)) {
939             return opener;
940         }
941 
942         if (TARGET_PARENT.equals(name)) {
943             return opener.getParentWindow();
944         }
945 
946         if (TARGET_TOP.equals(name)) {
947             return opener.getTopWindow();
948         }
949 
950         if (TARGET_BLANK.equals(name)) {
951             return null;
952         }
953 
954         // first search for frame windows inside our window hierarchy
955         WebWindow window = opener;
956         while (true) {
957             final Page page = window.getEnclosedPage();
958             if (page != null && page.isHtmlPage()) {
959                 try {
960                     final FrameWindow frame = ((HtmlPage) page).getFrameByName(name);
961                     final ScriptableObject scriptable = frame.getFrameElement().getScriptableObject();
962                     if (scriptable instanceof HTMLIFrameElement) {
963                         ((HTMLIFrameElement) scriptable).onRefresh();
964                     }
965                     return frame;
966                 }
967                 catch (final ElementNotFoundException e) {
968                     // Fall through
969                 }
970             }
971 
972             if (window == window.getParentWindow()) {
973                 // TODO: should getParentWindow() return null on top windows?
974                 break;
975             }
976             window = window.getParentWindow();
977         }
978 
979         try {
980             return getWebWindowByName(name);
981         }
982         catch (final WebWindowNotFoundException e) {
983             // Fall through - a new window will be created below
984         }
985         return null;
986     }
987 
988     /**
989      * <p><span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span></p>
990      *
991      * Opens a new dialog window.
992      * @param url the URL of the document to load and display
993      * @param opener the web window that is opening the dialog
994      * @param dialogArguments the object to make available inside the dialog via <tt>window.dialogArguments</tt>
995      * @return the new dialog window
996      * @throws IOException if there is an IO error
997      */
998     public DialogWindow openDialogWindow(final URL url, final WebWindow opener, final Object dialogArguments)
999         throws IOException {
1000 
1001         WebAssert.notNull("url", url);
1002         WebAssert.notNull("opener", opener);
1003 
1004         final DialogWindow window = new DialogWindow(this, dialogArguments);
1005         fireWindowOpened(new WebWindowEvent(window, WebWindowEvent.OPEN, null, null));
1006 
1007         final HtmlPage openerPage = (HtmlPage) opener.getEnclosedPage();
1008         final WebRequest request = new WebRequest(url, getBrowserVersion().getHtmlAcceptHeader());
1009         if (getBrowserVersion().hasFeature(DIALOGWINDOW_REFERER) && openerPage != null) {
1010             final String referer = openerPage.getUrl().toExternalForm();
1011             request.setAdditionalHeader("Referer", referer);
1012         }
1013 
1014         getPage(window, request);
1015 
1016         return window;
1017     }
1018 
1019     /**
1020      * Sets the object that will be used to create pages. Set this if you want
1021      * to customize the type of page that is returned for a given content type.
1022      *
1023      * @param pageCreator the new page creator
1024      */
1025     public void setPageCreator(final PageCreator pageCreator) {
1026         WebAssert.notNull("pageCreator", pageCreator);
1027         pageCreator_ = pageCreator;
1028     }
1029 
1030     /**
1031      * Returns the current page creator.
1032      *
1033      * @return the page creator
1034      */
1035     public PageCreator getPageCreator() {
1036         return pageCreator_;
1037     }
1038 
1039     /**
1040      * Returns the first {@link WebWindow} that matches the specified name.
1041      *
1042      * @param name the name to search for
1043      * @return the {@link WebWindow} with the specified name
1044      * @throws WebWindowNotFoundException if the {@link WebWindow} can't be found
1045      * @see #getWebWindows()
1046      * @see #getTopLevelWindows()
1047      */
1048     public WebWindow getWebWindowByName(final String name) throws WebWindowNotFoundException {
1049         WebAssert.notNull("name", name);
1050 
1051         for (final WebWindow webWindow : windows_) {
1052             if (name.equals(webWindow.getName())) {
1053                 return webWindow;
1054             }
1055         }
1056 
1057         throw new WebWindowNotFoundException(name);
1058     }
1059 
1060     /**
1061      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
1062      *
1063      * Initializes a new web window for JavaScript.
1064      * @param webWindow the new WebWindow
1065      */
1066     public void initialize(final WebWindow webWindow) {
1067         WebAssert.notNull("webWindow", webWindow);
1068         scriptEngine_.initialize(webWindow);
1069     }
1070 
1071     /**
1072      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
1073      *
1074      * Initializes a new page for JavaScript.
1075      * @param newPage the new page
1076      */
1077     public void initialize(final Page newPage) {
1078         WebAssert.notNull("newPage", newPage);
1079         final WebWindow webWindow = newPage.getEnclosingWindow();
1080         if (webWindow.getScriptableObject() instanceof Window) {
1081             ((Window) webWindow.getScriptableObject()).initialize(newPage);
1082         }
1083     }
1084 
1085     /**
1086      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
1087      *
1088      * Initializes a new empty window for JavaScript.
1089      *
1090      * @param webWindow the new WebWindow
1091      */
1092     public void initializeEmptyWindow(final WebWindow webWindow) {
1093         WebAssert.notNull("webWindow", webWindow);
1094         initialize(webWindow);
1095         ((Window) webWindow.getScriptableObject()).initialize();
1096     }
1097 
1098     /**
1099      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
1100      *
1101      * Adds a new window to the list of available windows.
1102      *
1103      * @param webWindow the new WebWindow
1104      */
1105     public void registerWebWindow(final WebWindow webWindow) {
1106         WebAssert.notNull("webWindow", webWindow);
1107         windows_.add(webWindow);
1108         // register JobManager here but don't deregister in deregisterWebWindow as it can live longer
1109         jobManagers_.add(new WeakReference<>(webWindow.getJobManager()));
1110     }
1111 
1112     /**
1113      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
1114      *
1115      * Removes a window from the list of available windows.
1116      *
1117      * @param webWindow the window to remove
1118      */
1119     public void deregisterWebWindow(final WebWindow webWindow) {
1120         WebAssert.notNull("webWindow", webWindow);
1121         if (windows_.remove(webWindow)) {
1122             fireWindowClosed(new WebWindowEvent(webWindow, WebWindowEvent.CLOSE, webWindow.getEnclosedPage(), null));
1123         }
1124     }
1125 
1126     /**
1127      * Expands a relative URL relative to the specified base. In most situations
1128      * this is the same as <code>new URL(baseUrl, relativeUrl)</code> but
1129      * there are some cases that URL doesn't handle correctly. See
1130      * <a href="http://www.faqs.org/rfcs/rfc1808.html">RFC1808</a>
1131      * regarding Relative Uniform Resource Locators for more information.
1132      *
1133      * @param baseUrl the base URL
1134      * @param relativeUrl the relative URL
1135      * @return the expansion of the specified base and relative URLs
1136      * @throws MalformedURLException if an error occurred when creating a URL object
1137      */
1138     public static URL expandUrl(final URL baseUrl, final String relativeUrl) throws MalformedURLException {
1139         final String newUrl = UrlUtils.resolveUrl(baseUrl, relativeUrl);
1140         return UrlUtils.toUrlUnsafe(newUrl);
1141     }
1142 
1143     private WebResponse makeWebResponseForDataUrl(final WebRequest webRequest) throws IOException {
1144         final URL url = webRequest.getUrl();
1145         final List<NameValuePair> responseHeaders = new ArrayList<>();
1146         final DataURLConnection connection;
1147         try {
1148             connection = new DataURLConnection(url);
1149         }
1150         catch (final DecoderException e) {
1151             throw new IOException(e.getMessage());
1152         }
1153         responseHeaders.add(new NameValuePair("content-type",
1154             connection.getMediaType() + ";charset=" + connection.getCharset()));
1155 
1156         try (InputStream is = connection.getInputStream()) {
1157             final DownloadedContent downloadedContent =
1158                     HttpWebConnection.downloadContent(is, getOptions().getMaxInMemory());
1159             final WebResponseData data = new WebResponseData(downloadedContent, 200, "OK", responseHeaders);
1160             return new WebResponse(data, url, webRequest.getHttpMethod(), 0);
1161         }
1162     }
1163 
1164     private static WebResponse makeWebResponseForAboutUrl(final URL url) {
1165         final String urlWithoutQuery = StringUtils.substringBefore(url.toExternalForm(), "?");
1166         if (!"blank".equalsIgnoreCase(StringUtils.substringAfter(urlWithoutQuery, WebClient.ABOUT_SCHEME))) {
1167             throw new IllegalArgumentException(url + " is not supported, only about:blank is supported now.");
1168         }
1169         return new StringWebResponse("", URL_ABOUT_BLANK);
1170     }
1171 
1172     /**
1173      * Builds a WebResponse for a file URL.
1174      * This first implementation is basic.
1175      * It assumes that the file contains an HTML page encoded with the specified encoding.
1176      * @param url the file URL
1177      * @param charset encoding to use
1178      * @return the web response
1179      * @throws IOException if an IO problem occurs
1180      */
1181     private WebResponse makeWebResponseForFileUrl(final WebRequest webRequest) throws IOException {
1182         URL cleanUrl = webRequest.getUrl();
1183         if (cleanUrl.getQuery() != null) {
1184             // Get rid of the query portion before trying to load the file.
1185             cleanUrl = UrlUtils.getUrlWithNewQuery(cleanUrl, null);
1186         }
1187         if (cleanUrl.getRef() != null) {
1188             // Get rid of the ref portion before trying to load the file.
1189             cleanUrl = UrlUtils.getUrlWithNewRef(cleanUrl, null);
1190         }
1191 
1192         String fileUrl = cleanUrl.toExternalForm();
1193         fileUrl = URLDecoder.decode(fileUrl, UTF_8.name());
1194         final File file = new File(fileUrl.substring(5));
1195         if (!file.exists()) {
1196             // construct 404
1197             final List<NameValuePair> compiledHeaders = new ArrayList<>();
1198             compiledHeaders.add(new NameValuePair("Content-Type", "text/html"));
1199             final WebResponseData responseData =
1200                 new WebResponseData(
1201                     TextUtil.stringToByteArray("File: " + file.getAbsolutePath(), UTF_8),
1202                     404, "Not Found", compiledHeaders);
1203             return new WebResponse(responseData, webRequest, 0);
1204         }
1205 
1206         final String contentType = guessContentType(file);
1207 
1208         final DownloadedContent content = new DownloadedContent.OnFile(file, false);
1209         final List<NameValuePair> compiledHeaders = new ArrayList<>();
1210         compiledHeaders.add(new NameValuePair("Content-Type", contentType));
1211         final WebResponseData responseData = new WebResponseData(content, 200, "OK", compiledHeaders);
1212         return new WebResponse(responseData, webRequest, 0);
1213     }
1214 
1215     /**
1216      * Tries to guess the content type of the file.<br>
1217      * This utility could be located in a helper class but we can compare this functionality
1218      * for instance with the "Helper Applications" settings of Mozilla and therefore see it as a
1219      * property of the "browser".
1220      * @param file the file
1221      * @return "application/octet-stream" if nothing could be guessed
1222      */
1223     public String guessContentType(final File file) {
1224         final String fileName = file.getName();
1225         if (fileName.endsWith(".xhtml")) {
1226             // Java's mime type map returns application/xml in JDK8.
1227             return "application/xhtml+xml";
1228         }
1229 
1230         // Java's mime type map does not know these in JDK8.
1231         if (fileName.endsWith(".js")) {
1232             return "text/javascript";
1233         }
1234         if (fileName.toLowerCase(Locale.ROOT).endsWith(".css")) {
1235             return "text/css";
1236         }
1237 
1238         String contentType = URLConnection.guessContentTypeFromName(fileName);
1239         if (contentType == null) {
1240             try (InputStream inputStream = new BufferedInputStream(new FileInputStream(file))) {
1241                 contentType = URLConnection.guessContentTypeFromStream(inputStream);
1242             }
1243             catch (final IOException e) {
1244                 // Ignore silently.
1245             }
1246         }
1247         if (contentType == null) {
1248             contentType = "application/octet-stream";
1249         }
1250         return contentType;
1251     }
1252 
1253     private WebResponse makeWebResponseForJavaScriptUrl(final WebWindow webWindow, final URL url,
1254         final Charset charset) throws FailingHttpStatusCodeException, IOException {
1255 
1256         HtmlPage page = null;
1257         if (webWindow instanceof FrameWindow) {
1258             final FrameWindow frameWindow = (FrameWindow) webWindow;
1259             page = (HtmlPage) frameWindow.getEnclosedPage();
1260         }
1261         else {
1262             Page currentPage = webWindow.getEnclosedPage();
1263             if (currentPage == null) {
1264                 // Starting with a JavaScript URL; quickly fill an "about:blank".
1265                 currentPage = getPage(webWindow, new WebRequest(WebClient.URL_ABOUT_BLANK));
1266             }
1267             else if (currentPage instanceof HtmlPage) {
1268                 page = (HtmlPage) currentPage;
1269             }
1270         }
1271 
1272         if (page == null) {
1273             page = getPage(webWindow, new WebRequest(WebClient.URL_ABOUT_BLANK));
1274         }
1275         final ScriptResult r = page.executeJavaScript(url.toExternalForm(), "JavaScript URL", 1);
1276         if (r.getJavaScriptResult() == null || ScriptResult.isUndefined(r)) {
1277             // No new WebResponse to produce.
1278             return webWindow.getEnclosedPage().getWebResponse();
1279         }
1280 
1281         final String contentString = r.getJavaScriptResult().toString();
1282         final StringWebResponse response = new StringWebResponse(contentString, charset, url);
1283         response.setFromJavascript(true);
1284         return response;
1285     }
1286 
1287     /**
1288      * Loads a {@link WebResponse} from the server.
1289      * @param webRequest the request
1290      * @throws IOException if an IO problem occurs
1291      * @return the WebResponse
1292      */
1293     public WebResponse loadWebResponse(final WebRequest webRequest) throws IOException {
1294         switch (webRequest.getUrl().getProtocol()) {
1295             case "about":
1296                 return makeWebResponseForAboutUrl(webRequest.getUrl());
1297 
1298             case "file":
1299                 return makeWebResponseForFileUrl(webRequest);
1300 
1301             case "data":
1302                 return makeWebResponseForDataUrl(webRequest);
1303 
1304             default:
1305                 return loadWebResponseFromWebConnection(webRequest, ALLOWED_REDIRECTIONS_SAME_URL,
1306                                                             webRequest.getCharset());
1307         }
1308     }
1309 
1310     /**
1311      * Loads a {@link WebResponse} from the server through the WebConnection.
1312      * @param webRequest the request
1313      * @param allowedRedirects the number of allowed redirects remaining
1314      * @param charset the charset to use
1315      * @throws IOException if an IO problem occurs
1316      * @return the resultant {@link WebResponse}
1317      */
1318     private WebResponse loadWebResponseFromWebConnection(final WebRequest webRequest,
1319         final int allowedRedirects, final Charset charset) throws IOException {
1320 
1321         URL url = webRequest.getUrl();
1322         final HttpMethod method = webRequest.getHttpMethod();
1323         final List<NameValuePair> parameters = webRequest.getRequestParameters();
1324 
1325         WebAssert.notNull("url", url);
1326         WebAssert.notNull("method", method);
1327         WebAssert.notNull("parameters", parameters);
1328 
1329         url = UrlUtils.encodeUrl(url, getBrowserVersion().hasFeature(URL_MINIMAL_QUERY_ENCODING), charset);
1330         webRequest.setUrl(url);
1331 
1332         if (LOG.isDebugEnabled()) {
1333             LOG.debug("Load response for " + method + " " + url.toExternalForm());
1334         }
1335 
1336         // If the request settings don't specify a custom proxy, use the default client proxy...
1337         if (webRequest.getProxyHost() == null) {
1338             final ProxyConfig proxyConfig = getOptions().getProxyConfig();
1339             if (proxyConfig.getProxyAutoConfigUrl() != null) {
1340                 if (!UrlUtils.sameFile(new URL(proxyConfig.getProxyAutoConfigUrl()), url)) {
1341                     String content = proxyConfig.getProxyAutoConfigContent();
1342                     if (content == null) {
1343                         content = getPage(proxyConfig.getProxyAutoConfigUrl())
1344                             .getWebResponse().getContentAsString();
1345                         proxyConfig.setProxyAutoConfigContent(content);
1346                     }
1347                     final String allValue = ProxyAutoConfig.evaluate(content, url);
1348                     if (LOG.isDebugEnabled()) {
1349                         LOG.debug("Proxy Auto-Config: value '" + allValue + "' for URL " + url);
1350                     }
1351                     String value = allValue.split(";")[0].trim();
1352                     if (value.startsWith("PROXY")) {
1353                         value = value.substring(6);
1354                         final int colonIndex = value.indexOf(':');
1355                         webRequest.setSocksProxy(false);
1356                         webRequest.setProxyHost(value.substring(0, colonIndex));
1357                         webRequest.setProxyPort(Integer.parseInt(value.substring(colonIndex + 1)));
1358                     }
1359                     else if (value.startsWith("SOCKS")) {
1360                         value = value.substring(6);
1361                         final int colonIndex = value.indexOf(':');
1362                         webRequest.setSocksProxy(true);
1363                         webRequest.setProxyHost(value.substring(0, colonIndex));
1364                         webRequest.setProxyPort(Integer.parseInt(value.substring(colonIndex + 1)));
1365                     }
1366                 }
1367             }
1368             // ...unless the host needs to bypass the configured client proxy!
1369             else if (!proxyConfig.shouldBypassProxy(webRequest.getUrl().getHost())) {
1370                 webRequest.setProxyHost(proxyConfig.getProxyHost());
1371                 webRequest.setProxyPort(proxyConfig.getProxyPort());
1372                 webRequest.setSocksProxy(proxyConfig.isSocksProxy());
1373             }
1374         }
1375 
1376         // Add the headers that are sent with every request.
1377         addDefaultHeaders(webRequest);
1378 
1379         // Retrieve the response, either from the cache or from the server.
1380         final WebResponse fromCache = getCache().getCachedResponse(webRequest);
1381         final WebResponse webResponse;
1382         if (fromCache != null) {
1383             webResponse = new WebResponseFromCache(fromCache, webRequest);
1384         }
1385         else {
1386             try {
1387                 webResponse = getWebConnection().getResponse(webRequest);
1388             }
1389             catch (final NoHttpResponseException e) {
1390                 return new WebResponse(responseDataNoHttpResponse_, webRequest, 0);
1391             }
1392             getCache().cacheIfPossible(webRequest, webResponse, null);
1393         }
1394 
1395         // Continue according to the HTTP status code.
1396         final int status = webResponse.getStatusCode();
1397         if (status == HttpStatus.SC_USE_PROXY) {
1398             getIncorrectnessListener().notify("Ignoring HTTP status code [305] 'Use Proxy'", this);
1399         }
1400         else if (status >= HttpStatus.SC_MOVED_PERMANENTLY
1401             && status <= (getBrowserVersion().hasFeature(HTTP_REDIRECT_308) ? 308 : 307)
1402             && status != HttpStatus.SC_NOT_MODIFIED
1403             && getOptions().isRedirectEnabled()) {
1404 
1405             final URL newUrl;
1406             String locationString = null;
1407             try {
1408                 locationString = webResponse.getResponseHeaderValue("Location");
1409                 if (locationString == null) {
1410                     return webResponse;
1411                 }
1412                 if (!getBrowserVersion().hasFeature(URL_MINIMAL_QUERY_ENCODING)) {
1413                     locationString = new String(locationString.getBytes(ISO_8859_1), UTF_8);
1414                 }
1415                 newUrl = expandUrl(url, locationString);
1416             }
1417             catch (final MalformedURLException e) {
1418                 getIncorrectnessListener().notify("Got a redirect status code [" + status + " "
1419                     + webResponse.getStatusMessage()
1420                     + "] but the location is not a valid URL [" + locationString
1421                     + "]. Skipping redirection processing.", this);
1422                 return webResponse;
1423             }
1424 
1425             if (LOG.isDebugEnabled()) {
1426                 LOG.debug("Got a redirect status code [" + status + "] new location = [" + locationString + "]");
1427             }
1428 
1429             if (allowedRedirects == 0) {
1430                 throw new FailingHttpStatusCodeException("Too much redirect for "
1431                     + webResponse.getWebRequest().getUrl(), webResponse);
1432             }
1433             else if (status == HttpStatus.SC_MOVED_PERMANENTLY
1434                     || status == HttpStatus.SC_MOVED_TEMPORARILY
1435                     || status == HttpStatus.SC_SEE_OTHER) {
1436                 final WebRequest wrs = new WebRequest(newUrl, HttpMethod.GET);
1437                 for (final Map.Entry<String, String> entry : webRequest.getAdditionalHeaders().entrySet()) {
1438                     wrs.setAdditionalHeader(entry.getKey(), entry.getValue());
1439                 }
1440                 return loadWebResponseFromWebConnection(wrs, allowedRedirects - 1, UTF_8);
1441             }
1442             else if (status == HttpStatus.SC_TEMPORARY_REDIRECT
1443                         || status == 308) {
1444                 final WebRequest wrs = new WebRequest(newUrl, webRequest.getHttpMethod());
1445                 wrs.setRequestParameters(parameters);
1446                 for (final Map.Entry<String, String> entry : webRequest.getAdditionalHeaders().entrySet()) {
1447                     wrs.setAdditionalHeader(entry.getKey(), entry.getValue());
1448                 }
1449                 return loadWebResponseFromWebConnection(wrs, allowedRedirects - 1, UTF_8);
1450             }
1451         }
1452 
1453         return webResponse;
1454     }
1455 
1456     /**
1457      * Adds the headers that are sent with every request to the specified {@link WebRequest} instance.
1458      * @param wrs the <tt>WebRequestSettings</tt> instance to modify
1459      */
1460     private void addDefaultHeaders(final WebRequest wrs) {
1461         // Add standard HtmlUnit headers.
1462         if (!wrs.isAdditionalHeader("Accept-Language")) {
1463             wrs.setAdditionalHeader("Accept-Language", getBrowserVersion().getBrowserLanguage());
1464         }
1465         if (getBrowserVersion().hasFeature(HTTP_HEADER_UPGRADE_INSECURE_REQUEST)
1466                 && !wrs.isAdditionalHeader("Upgrade-Insecure-Requests")) {
1467             wrs.setAdditionalHeader("Upgrade-Insecure-Requests", "1");
1468         }
1469         // Add user-specified headers last so that they can override HtmlUnit defaults.
1470         wrs.getAdditionalHeaders().putAll(requestHeaders_);
1471     }
1472 
1473     /**
1474      * Returns an immutable list of open web windows (whether they are top level windows or not).
1475      * This is a snapshot; future changes are not reflected by this list.
1476      *
1477      * @return an immutable list of open web windows (whether they are top level windows or not)
1478      * @see #getWebWindowByName(String)
1479      * @see #getTopLevelWindows()
1480      */
1481     public List<WebWindow> getWebWindows() {
1482         return Collections.unmodifiableList(new ArrayList<>(windows_));
1483     }
1484 
1485     /**
1486      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
1487      *
1488      * Returns true if the list of WebWindows contains the provided one.
1489      * This method is there to improve the performance of some internal checks because
1490      * calling getWebWindows().contains(.) creates some objects without any need.
1491      *
1492      * @param webWindow the window to check
1493      * @return true or false
1494      */
1495     public boolean containsWebWindow(final WebWindow webWindow) {
1496         return windows_.contains(webWindow);
1497     }
1498 
1499     /**
1500      * Returns an immutable list of open top level windows.
1501      * This is a snapshot; future changes are not reflected by this list.
1502      *
1503      * @return an immutable list of open top level windows
1504      * @see #getWebWindowByName(String)
1505      * @see #getWebWindows()
1506      */
1507     public List<TopLevelWindow> getTopLevelWindows() {
1508         return Collections.unmodifiableList(new ArrayList<>(topLevelWindows_));
1509     }
1510 
1511     /**
1512      * Sets the handler to be used whenever a refresh is triggered. Refer
1513      * to the documentation for {@link RefreshHandler} for more details.
1514      * @param handler the new handler
1515      */
1516     public void setRefreshHandler(final RefreshHandler handler) {
1517         if (handler == null) {
1518             refreshHandler_ = new NiceRefreshHandler(2);
1519         }
1520         else {
1521             refreshHandler_ = handler;
1522         }
1523     }
1524 
1525     /**
1526      * Returns the current refresh handler.
1527      * The default refresh handler is a {@link NiceRefreshHandler NiceRefreshHandler(2)}.
1528      * @return the current RefreshHandler
1529      */
1530     public RefreshHandler getRefreshHandler() {
1531         return refreshHandler_;
1532     }
1533 
1534     /**
1535      * Sets the script pre processor for this webclient.
1536      * @param scriptPreProcessor the new preprocessor or null if none is specified
1537      */
1538     public void setScriptPreProcessor(final ScriptPreProcessor scriptPreProcessor) {
1539         scriptPreProcessor_ = scriptPreProcessor;
1540     }
1541 
1542     /**
1543      * Returns the script pre processor for this webclient.
1544      * @return the pre processor or null of one hasn't been set
1545      */
1546     public ScriptPreProcessor getScriptPreProcessor() {
1547         return scriptPreProcessor_;
1548     }
1549 
1550     /**
1551      * Sets the active X object map for this webclient. The <code>Map</code> is used to map the
1552      * string passed into the <code>ActiveXObject</code> constructor to a java class name. Therefore
1553      * you can emulate <code>ActiveXObject</code>s in a web page's JavaScript by mapping the object
1554      * name to a java class to emulate the active X object.
1555      * @param activeXObjectMap the new preprocessor or null if none is specified
1556      */
1557     public void setActiveXObjectMap(final Map<String, String> activeXObjectMap) {
1558         activeXObjectMap_ = activeXObjectMap;
1559     }
1560 
1561     /**
1562      * Returns the active X object map for this webclient.
1563      * @return the active X object map
1564      */
1565     public Map<String, String> getActiveXObjectMap() {
1566         return activeXObjectMap_;
1567     }
1568 
1569     /**
1570      * Returns the MSXML ActiveX object factory (if supported).
1571      * @return the msxmlActiveXObjectFactory
1572      */
1573     public MSXMLActiveXObjectFactory getMSXMLActiveXObjectFactory() {
1574         return msxmlActiveXObjectFactory_;
1575     }
1576 
1577     /**
1578      * Sets the listener for messages generated by the HTML parser.
1579      * @param listener the new listener, {@code null} if messages should be totally ignored
1580      */
1581     public void setHTMLParserListener(final HTMLParserListener listener) {
1582         htmlParserListener_ = listener;
1583     }
1584 
1585     /**
1586      * Gets the configured listener for messages generated by the HTML parser.
1587      * @return {@code null} if no listener is defined (default value)
1588      */
1589     public HTMLParserListener getHTMLParserListener() {
1590         return htmlParserListener_;
1591     }
1592 
1593     /**
1594      * Returns the CSS error handler used by this web client when CSS problems are encountered.
1595      * @return the CSS error handler used by this web client when CSS problems are encountered
1596      * @see DefaultCssErrorHandler
1597      * @see SilentCssErrorHandler
1598      */
1599     public ErrorHandler getCssErrorHandler() {
1600         return cssErrorHandler_;
1601     }
1602 
1603     /**
1604      * Sets the CSS error handler used by this web client when CSS problems are encountered.
1605      * @param cssErrorHandler the CSS error handler used by this web client when CSS problems are encountered
1606      * @see DefaultCssErrorHandler
1607      * @see SilentCssErrorHandler
1608      */
1609     public void setCssErrorHandler(final ErrorHandler cssErrorHandler) {
1610         WebAssert.notNull("cssErrorHandler", cssErrorHandler);
1611         cssErrorHandler_ = cssErrorHandler;
1612     }
1613 
1614     /**
1615      * Sets the number of milliseconds that a script is allowed to execute before being terminated.
1616      * A value of 0 or less means no timeout.
1617      *
1618      * @param timeout the timeout value, in milliseconds
1619      */
1620     public void setJavaScriptTimeout(final long timeout) {
1621         scriptEngine_.setJavaScriptTimeout(timeout);
1622     }
1623 
1624     /**
1625      * Returns the number of milliseconds that a script is allowed to execute before being terminated.
1626      * A value of 0 or less means no timeout.
1627      *
1628      * @return the timeout value, in milliseconds
1629      */
1630     public long getJavaScriptTimeout() {
1631         return scriptEngine_.getJavaScriptTimeout();
1632     }
1633 
1634     /**
1635      * Gets the current listener for encountered incorrectness (except HTML parsing messages that
1636      * are handled by the HTML parser listener). Default value is an instance of
1637      * {@link IncorrectnessListenerImpl}.
1638      * @return the current listener (not {@code null})
1639      */
1640     public IncorrectnessListener getIncorrectnessListener() {
1641         return incorrectnessListener_;
1642     }
1643 
1644     /**
1645      * Returns the current HTML incorrectness listener.
1646      * @param listener the new value (not {@code null})
1647      */
1648     public void setIncorrectnessListener(final IncorrectnessListener listener) {
1649         if (listener == null) {
1650             throw new NullPointerException("Null incorrectness listener.");
1651         }
1652         incorrectnessListener_ = listener;
1653     }
1654 
1655     /**
1656      * Returns the WebConsole.
1657      * @return the web console
1658      */
1659     public WebConsole getWebConsole() {
1660         if (webConsole_ == null) {
1661             webConsole_ = new WebConsole();
1662         }
1663         return webConsole_;
1664     }
1665 
1666     /**
1667      * Gets the current AJAX controller.
1668      * @return the controller
1669      */
1670     public AjaxController getAjaxController() {
1671         return ajaxController_;
1672     }
1673 
1674     /**
1675      * Sets the current AJAX controller.
1676      * @param newValue the controller
1677      */
1678     public void setAjaxController(final AjaxController newValue) {
1679         if (newValue == null) {
1680             throw new NullPointerException();
1681         }
1682         ajaxController_ = newValue;
1683     }
1684 
1685     /**
1686      * Sets the attachment handler.
1687      * @param handler the new attachment handler
1688      */
1689     public void setAttachmentHandler(final AttachmentHandler handler) {
1690         attachmentHandler_ = handler;
1691     }
1692 
1693     /**
1694      * Returns the current attachment handler.
1695      * @return the current attachment handler
1696      */
1697     public AttachmentHandler getAttachmentHandler() {
1698         return attachmentHandler_;
1699     }
1700 
1701     /**
1702      * Sets the applet confirm handler.
1703      * @param handler the new applet confirm handler handler
1704      */
1705     public void setAppletConfirmHandler(final AppletConfirmHandler handler) {
1706         appletConfirmHandler_ = handler;
1707     }
1708 
1709     /**
1710      * Returns the current applet confirm handler.
1711      * @return the current applet confirm handler
1712      */
1713     public AppletConfirmHandler getAppletConfirmHandler() {
1714         return appletConfirmHandler_;
1715     }
1716 
1717     /**
1718      * Sets the onbeforeunload handler for this webclient.
1719      * @param onbeforeunloadHandler the new onbeforeunloadHandler or null if none is specified
1720      */
1721     public void setOnbeforeunloadHandler(final OnbeforeunloadHandler onbeforeunloadHandler) {
1722         onbeforeunloadHandler_ = onbeforeunloadHandler;
1723     }
1724 
1725     /**
1726      * Returns the onbeforeunload handler for this webclient.
1727      * @return the onbeforeunload handler or null if one hasn't been set
1728      */
1729     public OnbeforeunloadHandler getOnbeforeunloadHandler() {
1730         return onbeforeunloadHandler_;
1731     }
1732 
1733     /**
1734      * Gets the cache currently being used.
1735      * @return the cache (may not be null)
1736      */
1737     public Cache getCache() {
1738         return cache_;
1739     }
1740 
1741     /**
1742      * Sets the cache to use.
1743      * @param cache the new cache (must not be {@code null})
1744      */
1745     public void setCache(final Cache cache) {
1746         if (cache == null) {
1747             throw new IllegalArgumentException("cache should not be null!");
1748         }
1749         cache_ = cache;
1750     }
1751 
1752     /**
1753      * Keeps track of the current window. Inspired by WebTest's logic to track the current response.
1754      */
1755     private static final class CurrentWindowTracker implements WebWindowListener, Serializable {
1756         private final WebClient webClient_;
1757 
1758         private CurrentWindowTracker(final WebClient webClient) {
1759             webClient_ = webClient;
1760         }
1761 
1762         /**
1763          * {@inheritDoc}
1764          */
1765         @Override
1766         public void webWindowClosed(final WebWindowEvent event) {
1767             final WebWindow window = event.getWebWindow();
1768             if (window instanceof TopLevelWindow) {
1769                 webClient_.topLevelWindows_.remove(window);
1770                 if (window == webClient_.getCurrentWindow()) {
1771                     if (webClient_.topLevelWindows_.isEmpty()) {
1772                         // Must always have at least window, and there are no top-level windows left; must create one.
1773                         final TopLevelWindow newWindow = new TopLevelWindow("", webClient_);
1774                         webClient_.topLevelWindows_.push(newWindow);
1775                         webClient_.setCurrentWindow(newWindow);
1776                     }
1777                     else {
1778                         // The current window is now the previous top-level window.
1779                         webClient_.setCurrentWindow(webClient_.topLevelWindows_.peek());
1780                     }
1781                 }
1782             }
1783             else if (window == webClient_.getCurrentWindow()) {
1784                 // The current window is now the last top-level window.
1785                 webClient_.setCurrentWindow(webClient_.topLevelWindows_.peek());
1786             }
1787         }
1788 
1789         /**
1790          * {@inheritDoc}
1791          */
1792         @Override
1793         public void webWindowContentChanged(final WebWindowEvent event) {
1794             final WebWindow window = event.getWebWindow();
1795             boolean use = false;
1796             if (window instanceof DialogWindow) {
1797                 use = true;
1798             }
1799             else if (window instanceof TopLevelWindow) {
1800                 use = event.getOldPage() == null;
1801             }
1802             else if (window instanceof FrameWindow) {
1803                 final FrameWindow fw = (FrameWindow) window;
1804                 final String enclosingPageState = fw.getEnclosingPage().getDocumentElement().getReadyState();
1805                 final URL frameUrl = fw.getEnclosedPage().getUrl();
1806                 if (!DomNode.READY_STATE_COMPLETE.equals(enclosingPageState) || frameUrl == URL_ABOUT_BLANK) {
1807                     return;
1808                 }
1809 
1810                 // now looks at the visibility of the frame window
1811                 final BaseFrameElement frameElement = fw.getFrameElement();
1812                 if (frameElement.isDisplayed()) {
1813                     final Object element = frameElement.getScriptableObject();
1814                     final HTMLElement htmlElement = (HTMLElement) element;
1815                     final ComputedCSSStyleDeclaration style =
1816                             htmlElement.getWindow().getComputedStyle(htmlElement, null);
1817                     use = style.getCalculatedWidth(false, false) != 0
1818                             && style.getCalculatedHeight(false, false) != 0;
1819                 }
1820             }
1821             if (use) {
1822                 webClient_.setCurrentWindow(window);
1823             }
1824         }
1825 
1826         /**
1827          * {@inheritDoc}
1828          */
1829         @Override
1830         public void webWindowOpened(final WebWindowEvent event) {
1831             final WebWindow window = event.getWebWindow();
1832             if (window instanceof TopLevelWindow) {
1833                 final TopLevelWindow tlw = (TopLevelWindow) window;
1834                 webClient_.topLevelWindows_.push(tlw);
1835             }
1836             // Page is not loaded yet, don't set it now as current window.
1837         }
1838     }
1839 
1840     /**
1841      * Closes all opened windows, stopping all background JavaScript processing.
1842      *
1843      * {@inheritDoc}
1844      */
1845     @Override
1846     public void close() {
1847         // NB: this implementation is too simple as a new TopLevelWindow may be opened by
1848         // some JS script while we are closing the others
1849         final List<TopLevelWindow> topWindows = new ArrayList<>(topLevelWindows_);
1850         for (final TopLevelWindow topWindow : topWindows) {
1851             if (topLevelWindows_.contains(topWindow)) {
1852                 try {
1853                     topWindow.close();
1854                 }
1855                 catch (final Exception e) {
1856                     LOG.error("Exception while closing a topLevelWindow", e);
1857                 }
1858             }
1859         }
1860 
1861         // do this after closing the windows, otherwise some unload event might
1862         // start a new window that will start the thread again
1863         if (scriptEngine_ != null) {
1864             try {
1865                 scriptEngine_.shutdown();
1866             }
1867             catch (final Exception e) {
1868                 LOG.error("Exception while shutdown the scriptEngine", e);
1869             }
1870         }
1871 
1872         try {
1873             webConnection_.close();
1874         }
1875         catch (final Exception e) {
1876             LOG.error("Exception while closing the connection", e);
1877         }
1878 
1879         cache_.clear();
1880     }
1881 
1882     /**
1883      * <p><span style="color:red">Experimental API: May be changed in next release
1884      * and may not yet work perfectly!</span></p>
1885      *
1886      * <p>This method blocks until all background JavaScript tasks have finished executing. Background
1887      * JavaScript tasks are JavaScript tasks scheduled for execution via <tt>window.setTimeout</tt>,
1888      * <tt>window.setInterval</tt> or asynchronous <tt>XMLHttpRequest</tt>.</p>
1889      *
1890      * <p>If a job is scheduled to begin executing after <tt>(now + timeoutMillis)</tt>, this method will
1891      * wait for <tt>timeoutMillis</tt> milliseconds and then return a value greater than <tt>0</tt>. This
1892      * method will never block longer than <tt>timeoutMillis</tt> milliseconds.</p>
1893      *
1894      * <p>Use this method instead of {@link #waitForBackgroundJavaScriptStartingBefore(long)} if you
1895      * don't know when your background JavaScript is supposed to start executing, but you're fairly sure
1896      * that you know how long it should take to finish executing.</p>
1897      *
1898      * @param timeoutMillis the maximum amount of time to wait (in milliseconds)
1899      * @return the number of background JavaScript jobs still executing or waiting to be executed when this
1900      *         method returns; will be <tt>0</tt> if there are no jobs left to execute
1901      */
1902     public int waitForBackgroundJavaScript(final long timeoutMillis) {
1903         int count = 0;
1904         final long endTime = System.currentTimeMillis() + timeoutMillis;
1905         for (Iterator<WeakReference<JavaScriptJobManager>> i = jobManagers_.iterator(); i.hasNext();) {
1906             final JavaScriptJobManager jobManager;
1907             final WeakReference<JavaScriptJobManager> reference;
1908             try {
1909                 reference = i.next();
1910                 jobManager = reference.get();
1911                 if (jobManager == null) {
1912                     i.remove();
1913                     continue;
1914                 }
1915             }
1916             catch (final ConcurrentModificationException e) {
1917                 i = jobManagers_.iterator();
1918                 count = 0;
1919                 continue;
1920             }
1921 
1922             final long newTimeout = endTime - System.currentTimeMillis();
1923             count += jobManager.waitForJobs(newTimeout);
1924         }
1925         if (count != getAggregateJobCount()) {
1926             final long newTimeout = endTime - System.currentTimeMillis();
1927             return waitForBackgroundJavaScript(newTimeout);
1928         }
1929         return count;
1930     }
1931 
1932     /**
1933      * <p><span style="color:red">Experimental API: May be changed in next release
1934      * and may not yet work perfectly!</span></p>
1935      *
1936      * <p>This method blocks until all background JavaScript tasks scheduled to start executing before
1937      * <tt>(now + delayMillis)</tt> have finished executing. Background JavaScript tasks are JavaScript
1938      * tasks scheduled for execution via <tt>window.setTimeout</tt>, <tt>window.setInterval</tt> or
1939      * asynchronous <tt>XMLHttpRequest</tt>.</p>
1940      *
1941      * <p>If there is no background JavaScript task currently executing, and there is no background JavaScript
1942      * task scheduled to start executing within the specified time, this method returns immediately -- even
1943      * if there are tasks scheduled to be executed after <tt>(now + delayMillis)</tt>.</p>
1944      *
1945      * <p>Note that the total time spent executing a background JavaScript task is never known ahead of
1946      * time, so this method makes no guarantees as to how long it will block.</p>
1947      *
1948      * <p>Use this method instead of {@link #waitForBackgroundJavaScript(long)} if you know roughly when
1949      * your background JavaScript is supposed to start executing, but you're not necessarily sure how long
1950      * it will take to execute.</p>
1951      *
1952      * @param delayMillis the delay which determines the background tasks to wait for (in milliseconds)
1953      * @return the number of background JavaScript jobs still executing or waiting to be executed when this
1954      *         method returns; will be <tt>0</tt> if there are no jobs left to execute
1955      */
1956     public int waitForBackgroundJavaScriptStartingBefore(final long delayMillis) {
1957         int count = 0;
1958         final long endTime = System.currentTimeMillis() + delayMillis;
1959         for (Iterator<WeakReference<JavaScriptJobManager>> i = jobManagers_.iterator(); i.hasNext();) {
1960             final JavaScriptJobManager jobManager;
1961             final WeakReference<JavaScriptJobManager> reference;
1962             try {
1963                 reference = i.next();
1964                 jobManager = reference.get();
1965                 if (jobManager == null) {
1966                     i.remove();
1967                     continue;
1968                 }
1969             }
1970             catch (final ConcurrentModificationException e) {
1971                 i = jobManagers_.iterator();
1972                 count = 0;
1973                 continue;
1974             }
1975             final long newDelay = endTime - System.currentTimeMillis();
1976             count += jobManager.waitForJobsStartingBefore(newDelay);
1977         }
1978         if (count != getAggregateJobCount()) {
1979             final long newDelay = endTime - System.currentTimeMillis();
1980             return waitForBackgroundJavaScriptStartingBefore(newDelay);
1981         }
1982         return count;
1983     }
1984 
1985     /**
1986      * Returns the aggregate background JavaScript job count across all windows.
1987      * @return the aggregate background JavaScript job count across all windows
1988      */
1989     private int getAggregateJobCount() {
1990         int count = 0;
1991         for (Iterator<WeakReference<JavaScriptJobManager>> i = jobManagers_.iterator(); i.hasNext();) {
1992             final JavaScriptJobManager jobManager;
1993             final WeakReference<JavaScriptJobManager> reference;
1994             try {
1995                 reference = i.next();
1996                 jobManager = reference.get();
1997                 if (jobManager == null) {
1998                     i.remove();
1999                     continue;
2000                 }
2001             }
2002             catch (final ConcurrentModificationException e) {
2003                 i = jobManagers_.iterator();
2004                 count = 0;
2005                 continue;
2006             }
2007             final int jobCount = jobManager.getJobCount();
2008             count += jobCount;
2009         }
2010         return count;
2011     }
2012 
2013     /**
2014      * When we deserialize, re-initializie transient fields.
2015      * @param in the object input stream
2016      * @throws IOException if an error occurs
2017      * @throws ClassNotFoundException if an error occurs
2018      */
2019     private void readObject(final ObjectInputStream in) throws IOException, ClassNotFoundException {
2020         in.defaultReadObject();
2021 
2022         webConnection_ = createWebConnection();
2023         scriptEngine_ = new JavaScriptEngine(this);
2024         jobManagers_ = Collections.synchronizedList(new ArrayList<WeakReference<JavaScriptJobManager>>());
2025 
2026         if (getBrowserVersion().hasFeature(JS_XML_SUPPORT_VIA_ACTIVEXOBJECT)) {
2027             initMSXMLActiveX();
2028         }
2029     }
2030 
2031     private WebConnection createWebConnection() {
2032         if (GAEUtils.isGaeMode()) {
2033             return new UrlFetchWebConnection(this);
2034         }
2035 
2036         return new HttpWebConnection(this);
2037     }
2038 
2039     private static class LoadJob {
2040         private final WebWindow requestingWindow_;
2041         private final String target_;
2042         private final WebResponse response_;
2043         private final URL urlWithOnlyHashChange_;
2044         private final WeakReference<Page> originalPage_;
2045         private final WebRequest request_;
2046 
2047         LoadJob(final WebRequest request, final WebWindow requestingWindow, final String target,
2048                 final WebResponse response) {
2049             request_ = request;
2050             requestingWindow_ = requestingWindow;
2051             target_ = target;
2052             response_ = response;
2053             urlWithOnlyHashChange_ = null;
2054             originalPage_ = new WeakReference<>(requestingWindow.getEnclosedPage());
2055         }
2056 
2057         LoadJob(final WebRequest request, final WebWindow requestingWindow, final String target,
2058                 final URL urlWithOnlyHashChange) {
2059             request_ = request;
2060             requestingWindow_ = requestingWindow;
2061             target_ = target;
2062             response_ = null;
2063             urlWithOnlyHashChange_ = urlWithOnlyHashChange;
2064             originalPage_ = new WeakReference<>(requestingWindow.getEnclosedPage());
2065         }
2066 
2067         public boolean isOutdated() {
2068             if (target_ != null && !target_.isEmpty()) {
2069                 return false;
2070             }
2071             else if (requestingWindow_.isClosed()) {
2072                 return true;
2073             }
2074             else if (requestingWindow_.getEnclosedPage() != originalPage_.get()) {
2075                 return true;
2076             }
2077 
2078             return false;
2079         }
2080     }
2081 
2082     private final List<LoadJob> loadQueue_ = new ArrayList<>();
2083 
2084     /**
2085      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2086      *
2087      * Perform the downloads and stores it for loading later into a window.
2088      * In the future downloads should be performed in parallel in separated threads.
2089      * TODO: refactor it before next release.
2090      * @param requestingWindow the window from which the request comes
2091      * @param target the name of the target window
2092      * @param request the request to perform
2093      * @param checkHash if true check for hashChenage
2094      * @param forceLoad if true always load the request even if there is already the same in the queue
2095      * @param description information about the origin of the request. Useful for debugging.
2096      */
2097     public void download(final WebWindow requestingWindow, final String target,
2098         final WebRequest request, final boolean checkHash, final boolean forceLoad, final String description) {
2099         final WebWindow win = resolveWindow(requestingWindow, target);
2100         final URL url = request.getUrl();
2101         boolean justHashJump = false;
2102 
2103         if (win != null && HttpMethod.POST != request.getHttpMethod()) {
2104             final Page page = win.getEnclosedPage();
2105             if (page != null) {
2106                 if (page.isHtmlPage() && !((HtmlPage) page).isOnbeforeunloadAccepted()) {
2107                     return;
2108                 }
2109 
2110                 if (checkHash) {
2111                     final URL current = page.getUrl();
2112                     justHashJump =
2113                             HttpMethod.GET == request.getHttpMethod()
2114                             && UrlUtils.sameFile(url, current)
2115                             && null != url.getRef();
2116                 }
2117             }
2118         }
2119 
2120         synchronized (loadQueue_) {
2121             // verify if this load job doesn't already exist
2122             for (final LoadJob loadJob : loadQueue_) {
2123                 if (loadJob.response_ == null) {
2124                     continue;
2125                 }
2126                 final WebRequest otherRequest = loadJob.request_;
2127                 final URL otherUrl = otherRequest.getUrl();
2128 
2129                 // TODO: investigate but it seems that IE considers query string too but not FF
2130                 if (!forceLoad
2131                     && url.getPath().equals(otherUrl.getPath()) // fail fast
2132                     && url.toString().equals(otherUrl.toString())
2133                     && request.getRequestParameters().equals(otherRequest.getRequestParameters())
2134                     && StringUtils.equals(request.getRequestBody(), otherRequest.getRequestBody())) {
2135                     return; // skip it;
2136                 }
2137             }
2138         }
2139 
2140         final LoadJob loadJob;
2141         if (justHashJump) {
2142             loadJob = new LoadJob(request, requestingWindow, target, url);
2143         }
2144         else {
2145             try {
2146                 final WebResponse response = loadWebResponse(request);
2147                 loadJob = new LoadJob(request, requestingWindow, target, response);
2148             }
2149             catch (final IOException e) {
2150                 throw new RuntimeException(e);
2151             }
2152         }
2153         synchronized (loadQueue_) {
2154             loadQueue_.add(loadJob);
2155         }
2156     }
2157 
2158     /**
2159      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2160      *
2161      * Loads downloaded responses into the corresponding windows.
2162      * TODO: refactor it before next release.
2163      * @throws IOException in case of exception
2164      * @throws FailingHttpStatusCodeException in case of exception
2165      */
2166     public void loadDownloadedResponses() throws FailingHttpStatusCodeException, IOException {
2167         final List<LoadJob> queue;
2168 
2169         // synchronize access to the loadQueue_,
2170         // to be sure no job is ignored
2171         synchronized (loadQueue_) {
2172             if (loadQueue_.isEmpty()) {
2173                 return;
2174             }
2175             queue = new ArrayList<>(loadQueue_);
2176             loadQueue_.clear();
2177         }
2178 
2179         final HashSet<WebWindow> updatedWindows = new HashSet<>();
2180         for (int i = queue.size() - 1; i >= 0; --i) {
2181             final LoadJob loadJob = queue.get(i);
2182             if (loadJob.isOutdated()) {
2183                 LOG.info("No usage of download: " + loadJob);
2184                 continue;
2185             }
2186 
2187             final WebWindow window = resolveWindow(loadJob.requestingWindow_, loadJob.target_);
2188             if (!updatedWindows.contains(window)) {
2189                 final WebWindow win = openTargetWindow(loadJob.requestingWindow_, loadJob.target_, "_self");
2190                 if (loadJob.urlWithOnlyHashChange_ != null) {
2191                     final HtmlPage page = (HtmlPage) loadJob.requestingWindow_.getEnclosedPage();
2192                     final String oldURL = page.getUrl().toExternalForm();
2193 
2194                     // update request url
2195                     final WebRequest req = page.getWebResponse().getWebRequest();
2196                     req.setUrl(loadJob.urlWithOnlyHashChange_);
2197 
2198                     // update location.hash
2199                     final Window jsWindow = (Window) win.getScriptableObject();
2200                     if (null != jsWindow) {
2201                         final Location location = jsWindow.getLocation();
2202                         location.setHash(oldURL, loadJob.urlWithOnlyHashChange_.getRef());
2203                     }
2204 
2205                     // add to history
2206                     win.getHistory().addPage(page);
2207                 }
2208                 else {
2209                     final Page pageBeforeLoad = win.getEnclosedPage();
2210                     loadWebResponseInto(loadJob.response_, win);
2211 
2212                     // start execution here.
2213                     if (scriptEngine_ != null) {
2214                         scriptEngine_.registerWindowAndMaybeStartEventLoop(win);
2215                     }
2216 
2217                     if (pageBeforeLoad != win.getEnclosedPage()) {
2218                         updatedWindows.add(win);
2219                     }
2220 
2221                     // check and report problems if needed
2222                     throwFailingHttpStatusCodeExceptionIfNecessary(loadJob.response_);
2223                 }
2224             }
2225             else {
2226                 LOG.info("No usage of download: " + loadJob);
2227             }
2228         }
2229     }
2230 
2231     /**
2232      * Returns the options object of this WebClient.
2233      * @return the options object
2234      */
2235     public WebClientOptions getOptions() {
2236         return options_;
2237     }
2238 
2239     /**
2240      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2241      *
2242      * Returns the internals object of this WebClient.
2243      * @return the internals object
2244      */
2245     public WebClientInternals getInternals() {
2246         return internals_;
2247     }
2248 
2249     /**
2250      * Gets the holder for the different storages.
2251      * <p><span style="color:red">Experimental API: May be changed in next release!</span></p>
2252      * @return the holder
2253      */
2254     public StorageHolder getStorageHolder() {
2255         return storageHolder_;
2256     }
2257 
2258     /**
2259      * Returns the currently configured cookies applicable to the specified URL, in an unmodifiable set.
2260      * If disabled, this returns an empty set.
2261      * @param url the URL on which to filter the returned cookies
2262      * @return the currently configured cookies applicable to the specified URL, in an unmodifiable set
2263      */
2264     public synchronized Set<Cookie> getCookies(final URL url) {
2265         final CookieManager cookieManager = getCookieManager();
2266 
2267         if (!cookieManager.isCookiesEnabled()) {
2268             return Collections.<Cookie>emptySet();
2269         }
2270 
2271         final URL normalizedUrl = cookieManager.replaceForCookieIfNecessary(url);
2272 
2273         final String host = normalizedUrl.getHost();
2274         // URLs like "about:blank" don't have cookies and we need to catch these
2275         // cases here before HttpClient complains
2276         if (host.isEmpty()) {
2277             return Collections.emptySet();
2278         }
2279 
2280         final String path = normalizedUrl.getPath();
2281         final String protocol = normalizedUrl.getProtocol();
2282         final boolean secure = "https".equals(protocol);
2283 
2284         final int port = cookieManager.getPort(normalizedUrl);
2285 
2286         // discard expired cookies
2287         cookieManager.clearExpired(new Date());
2288 
2289         final List<org.apache.http.cookie.Cookie> all = Cookie.toHttpClient(cookieManager.getCookies());
2290         final List<org.apache.http.cookie.Cookie> matches = new ArrayList<>();
2291 
2292         if (all.size() > 0) {
2293             final CookieOrigin cookieOrigin = new CookieOrigin(host, port, path, secure);
2294             final CookieSpec cookieSpec = new HtmlUnitBrowserCompatCookieSpec(getBrowserVersion());
2295             for (final org.apache.http.cookie.Cookie cookie : all) {
2296                 if (cookieSpec.match(cookie, cookieOrigin)) {
2297                     matches.add(cookie);
2298                 }
2299             }
2300         }
2301 
2302         final Set<Cookie> cookies = new LinkedHashSet<>();
2303         cookies.addAll(Cookie.fromHttpClient(matches));
2304         return Collections.unmodifiableSet(cookies);
2305     }
2306 
2307     /**
2308      * Parses the given cookie and adds this to our cookie store.
2309      * @param cookieString the string to parse
2310      * @param pageUrl the url of the page that likes to set the cookie
2311      * @param origin the requester
2312      */
2313     public void addCookie(final String cookieString, final URL pageUrl, final Object origin) {
2314         final BrowserVersion browserVersion = getBrowserVersion();
2315         final CookieManager cookieManager = getCookieManager();
2316         if (cookieManager.isCookiesEnabled()) {
2317             final CharArrayBuffer buffer = new CharArrayBuffer(cookieString.length() + 22);
2318             buffer.append("Set-Cookie: ");
2319             buffer.append(cookieString);
2320 
2321             final CookieSpec cookieSpec = new HtmlUnitBrowserCompatCookieSpec(browserVersion);
2322 
2323             try {
2324                 final List<org.apache.http.cookie.Cookie> cookies =
2325                         cookieSpec.parse(new BufferedHeader(buffer), cookieManager.buildCookieOrigin(pageUrl));
2326 
2327                 for (org.apache.http.cookie.Cookie cookie : cookies) {
2328                     final Cookie htmlUnitCookie = new Cookie((ClientCookie) cookie);
2329                     cookieManager.addCookie(htmlUnitCookie);
2330 
2331                     if (LOG.isDebugEnabled()) {
2332                         LOG.debug("Added cookie: '" + cookieString + "'");
2333                     }
2334                 }
2335             }
2336             catch (final MalformedCookieException e) {
2337                 getIncorrectnessListener().notify("set-cookie http-equiv meta tag: invalid cookie '"
2338                         + cookieString + "'; reason: '" + e.getMessage() + "'.", origin);
2339             }
2340         }
2341         else if (LOG.isDebugEnabled()) {
2342             LOG.debug("Skipped adding cookie: '" + cookieString + "'");
2343         }
2344     }
2345 }