View Javadoc
1   /*
2    * Copyright (c) 2002-2017 Gargoyle Software Inc.
3    *
4    * Licensed under the Apache License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    * http://www.apache.org/licenses/LICENSE-2.0
8    *
9    * Unless required by applicable law or agreed to in writing, software
10   * distributed under the License is distributed on an "AS IS" BASIS,
11   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12   * See the License for the specific language governing permissions and
13   * limitations under the License.
14   */
15  package com.gargoylesoftware.htmlunit;
16  
17  import static com.gargoylesoftware.htmlunit.BrowserVersionFeatures.DIALOGWINDOW_REFERER;
18  import static com.gargoylesoftware.htmlunit.BrowserVersionFeatures.HTTP_HEADER_UPGRADE_INSECURE_REQUEST;
19  import static com.gargoylesoftware.htmlunit.BrowserVersionFeatures.HTTP_REDIRECT_308;
20  import static com.gargoylesoftware.htmlunit.BrowserVersionFeatures.JS_XML_SUPPORT_VIA_ACTIVEXOBJECT;
21  import static com.gargoylesoftware.htmlunit.BrowserVersionFeatures.URL_MINIMAL_QUERY_ENCODING;
22  import static com.gargoylesoftware.htmlunit.BrowserVersionFeatures.WINDOW_EXECUTE_EVENTS;
23  import static java.nio.charset.StandardCharsets.ISO_8859_1;
24  import static java.nio.charset.StandardCharsets.UTF_8;
25  
26  import java.io.BufferedInputStream;
27  import java.io.File;
28  import java.io.FileInputStream;
29  import java.io.IOException;
30  import java.io.InputStream;
31  import java.io.ObjectInputStream;
32  import java.io.Serializable;
33  import java.lang.ref.WeakReference;
34  import java.net.MalformedURLException;
35  import java.net.URL;
36  import java.net.URLConnection;
37  import java.net.URLDecoder;
38  import java.nio.charset.Charset;
39  import java.util.ArrayList;
40  import java.util.Collections;
41  import java.util.ConcurrentModificationException;
42  import java.util.Date;
43  import java.util.HashMap;
44  import java.util.HashSet;
45  import java.util.Iterator;
46  import java.util.LinkedHashSet;
47  import java.util.List;
48  import java.util.Locale;
49  import java.util.Map;
50  import java.util.Set;
51  import java.util.Stack;
52  
53  import org.apache.commons.codec.DecoderException;
54  import org.apache.commons.lang3.StringUtils;
55  import org.apache.commons.logging.Log;
56  import org.apache.commons.logging.LogFactory;
57  import org.apache.http.HttpStatus;
58  import org.apache.http.NoHttpResponseException;
59  import org.apache.http.client.CredentialsProvider;
60  import org.apache.http.cookie.ClientCookie;
61  import org.apache.http.cookie.CookieOrigin;
62  import org.apache.http.cookie.CookieSpec;
63  import org.apache.http.cookie.MalformedCookieException;
64  import org.apache.http.message.BufferedHeader;
65  import org.apache.http.util.CharArrayBuffer;
66  import org.w3c.css.sac.ErrorHandler;
67  
68  import com.gargoylesoftware.htmlunit.activex.javascript.msxml.MSXMLActiveXObjectFactory;
69  import com.gargoylesoftware.htmlunit.attachment.Attachment;
70  import com.gargoylesoftware.htmlunit.attachment.AttachmentHandler;
71  import com.gargoylesoftware.htmlunit.gae.GAEUtils;
72  import com.gargoylesoftware.htmlunit.html.BaseFrameElement;
73  import com.gargoylesoftware.htmlunit.html.DomElement;
74  import com.gargoylesoftware.htmlunit.html.DomNode;
75  import com.gargoylesoftware.htmlunit.html.FrameWindow;
76  import com.gargoylesoftware.htmlunit.html.HTMLParserListener;
77  import com.gargoylesoftware.htmlunit.html.HtmlInlineFrame;
78  import com.gargoylesoftware.htmlunit.html.HtmlPage;
79  import com.gargoylesoftware.htmlunit.httpclient.HtmlUnitBrowserCompatCookieSpec;
80  import com.gargoylesoftware.htmlunit.javascript.AbstractJavaScriptEngine;
81  import com.gargoylesoftware.htmlunit.javascript.DefaultJavaScriptErrorListener;
82  import com.gargoylesoftware.htmlunit.javascript.JavaScriptEngine;
83  import com.gargoylesoftware.htmlunit.javascript.JavaScriptErrorListener;
84  import com.gargoylesoftware.htmlunit.javascript.background.JavaScriptJobManager;
85  import com.gargoylesoftware.htmlunit.javascript.host.Location;
86  import com.gargoylesoftware.htmlunit.javascript.host.Window;
87  import com.gargoylesoftware.htmlunit.javascript.host.css.ComputedCSSStyleDeclaration;
88  import com.gargoylesoftware.htmlunit.javascript.host.dom.Node;
89  import com.gargoylesoftware.htmlunit.javascript.host.event.Event;
90  import com.gargoylesoftware.htmlunit.javascript.host.html.HTMLDocument;
91  import com.gargoylesoftware.htmlunit.javascript.host.html.HTMLElement;
92  import com.gargoylesoftware.htmlunit.javascript.host.html.HTMLIFrameElement;
93  import com.gargoylesoftware.htmlunit.protocol.data.DataURLConnection;
94  import com.gargoylesoftware.htmlunit.util.Cookie;
95  import com.gargoylesoftware.htmlunit.util.NameValuePair;
96  import com.gargoylesoftware.htmlunit.util.UrlUtils;
97  import com.gargoylesoftware.htmlunit.webstart.WebStartHandler;
98  
99  import net.sourceforge.htmlunit.corejs.javascript.ScriptableObject;
100 
101 /**
102  * The main starting point in HtmlUnit: this class simulates a web browser.
103  * <p>
104  * A standard usage of HtmlUnit will start with using the {@link #getPage(String)} method
105  * (or {@link #getPage(URL)}) to load a first {@link Page}
106  * and will continue with further processing on this page depending on its type.
107  * </p>
108  * <b>Example:</b><br>
109  * <br>
110  * <code>
111  * final WebClient webClient = new WebClient();<br>
112  * final {@link HtmlPage} startPage = webClient.getPage("http://htmlunit.sf.net");<br>
113  * assertEquals("HtmlUnit - Welcome to HtmlUnit", startPage.{@link HtmlPage#getTitleText() getTitleText}());
114  * </code>
115  * <p>
116  * Note: a {@link WebClient} instance is <b>not thread safe</b>. It is intended to be used from a single thread.
117  * </p>
118  * @author <a href="mailto:mbowler@GargoyleSoftware.com">Mike Bowler</a>
119  * @author <a href="mailto:gudujarlson@sf.net">Mike J. Bresnahan</a>
120  * @author Dominique Broeglin
121  * @author Noboru Sinohara
122  * @author <a href="mailto:chen_jun@users.sourceforge.net">Chen Jun</a>
123  * @author David K. Taylor
124  * @author <a href="mailto:cse@dynabean.de">Christian Sell</a>
125  * @author <a href="mailto:bcurren@esomnie.com">Ben Curren</a>
126  * @author Marc Guillemot
127  * @author Chris Erskine
128  * @author Daniel Gredler
129  * @author Sergey Gorelkin
130  * @author Hans Donner
131  * @author Paul King
132  * @author Ahmed Ashour
133  * @author Bruce Chapman
134  * @author Sudhan Moghe
135  * @author Martin Tamme
136  * @author Amit Manjhi
137  * @author Nicolas Belisle
138  * @author Ronald Brill
139  * @author Frank Danek
140  * @author Joerg Werner
141  */
142 public class WebClient implements Serializable, AutoCloseable {
143 
144     /** Logging support. */
145     private static final Log LOG = LogFactory.getLog(WebClient.class);
146 
147     /** Like the Firefox default value for {@code network.http.redirection-limit}. */
148     private static final int ALLOWED_REDIRECTIONS_SAME_URL = 20;
149 
150     private transient WebConnection webConnection_;
151     private CredentialsProvider credentialsProvider_ = new DefaultCredentialsProvider();
152     private CookieManager cookieManager_ = new CookieManager();
153     private transient AbstractJavaScriptEngine<?> scriptEngine_;
154     private final Map<String, String> requestHeaders_ = Collections.synchronizedMap(new HashMap<String, String>(89));
155     private IncorrectnessListener incorrectnessListener_ = new IncorrectnessListenerImpl();
156     private WebConsole webConsole_;
157 
158     private AlertHandler alertHandler_;
159     private ConfirmHandler confirmHandler_;
160     private PromptHandler promptHandler_;
161     private StatusHandler statusHandler_;
162     private AttachmentHandler attachmentHandler_;
163     private WebStartHandler webStartHandler_;
164     private AppletConfirmHandler appletConfirmHandler_;
165     private AjaxController ajaxController_ = new AjaxController();
166 
167     private BrowserVersion browserVersion_;
168     private PageCreator pageCreator_ = new DefaultPageCreator();
169 
170     private final Set<WebWindowListener> webWindowListeners_ = new HashSet<>(5);
171     private final Stack<TopLevelWindow> topLevelWindows_ = new Stack<>(); // top-level windows
172     private final List<WebWindow> windows_ = Collections.synchronizedList(new ArrayList<WebWindow>()); // all windows
173     private transient List<WeakReference<JavaScriptJobManager>> jobManagers_ =
174             Collections.synchronizedList(new ArrayList<WeakReference<JavaScriptJobManager>>());
175     private WebWindow currentWindow_;
176 
177     private HTMLParserListener htmlParserListener_;
178     private ErrorHandler cssErrorHandler_ = new DefaultCssErrorHandler();
179     private OnbeforeunloadHandler onbeforeunloadHandler_;
180     private Cache cache_ = new Cache();
181 
182     /** target "_blank". */
183     private static final String TARGET_BLANK = "_blank";
184     /** target "_parent". */
185     private static final String TARGET_SELF = "_self";
186     /** target "_parent". */
187     private static final String TARGET_PARENT = "_parent";
188     /** target "_top". */
189     private static final String TARGET_TOP = "_top";
190 
191     /** "about:". */
192     public static final String ABOUT_SCHEME = "about:";
193     /** "about:blank". */
194     public static final String ABOUT_BLANK = ABOUT_SCHEME + "blank";
195     /** URL for "about:blank". */
196     public static final URL URL_ABOUT_BLANK = UrlUtils.toUrlSafe(ABOUT_BLANK);
197 
198     private ScriptPreProcessor scriptPreProcessor_;
199 
200     private Map<String, String> activeXObjectMap_ = Collections.emptyMap();
201     private transient MSXMLActiveXObjectFactory msxmlActiveXObjectFactory_;
202     private RefreshHandler refreshHandler_ = new NiceRefreshHandler(2);
203     private JavaScriptErrorListener javaScriptErrorListener_ = new DefaultJavaScriptErrorListener();
204 
205     private WebClientOptions options_ = new WebClientOptions();
206     private WebClientInternals internals_ = new WebClientInternals(this);
207     private final StorageHolder storageHolder_ = new StorageHolder();
208 
209     private static final WebResponseData responseDataNoHttpResponse_ = new WebResponseData(
210         0, "No HTTP Response", Collections.<NameValuePair>emptyList());
211 
212     /**
213      * Creates a web client instance using the browser version returned by
214      * {@link BrowserVersion#getDefault()}.
215      */
216     public WebClient() {
217         this(BrowserVersion.getDefault());
218     }
219 
220     /**
221      * Creates a web client instance using the specified {@link BrowserVersion}.
222      * @param browserVersion the browser version to simulate
223      */
224     public WebClient(final BrowserVersion browserVersion) {
225         WebAssert.notNull("browserVersion", browserVersion);
226         init(browserVersion, new ProxyConfig());
227     }
228 
229     /**
230      * Creates an instance that will use the specified {@link BrowserVersion} and proxy server.
231      * @param browserVersion the browser version to simulate
232      * @param proxyHost the server that will act as proxy
233      * @param proxyPort the port to use on the proxy server
234      */
235     public WebClient(final BrowserVersion browserVersion, final String proxyHost, final int proxyPort) {
236         WebAssert.notNull("browserVersion", browserVersion);
237         WebAssert.notNull("proxyHost", proxyHost);
238         init(browserVersion, new ProxyConfig(proxyHost, proxyPort));
239     }
240 
241     /**
242      * Generic initialization logic used by all constructors. This method does not perform any
243      * parameter validation; such validation must be handled by the constructors themselves.
244      * @param browserVersion the browser version to simulate
245      * @param proxyConfig the proxy configuration to use
246      */
247     private void init(final BrowserVersion browserVersion, final ProxyConfig proxyConfig) {
248         browserVersion_ = browserVersion;
249         getOptions().setProxyConfig(proxyConfig);
250 
251         webConnection_ = createWebConnection(); // this has to be done after the browser version was set
252         scriptEngine_ = new JavaScriptEngine(this);
253         // The window must be constructed AFTER the script engine.
254         addWebWindowListener(new CurrentWindowTracker(this));
255         currentWindow_ = new TopLevelWindow("", this);
256         fireWindowOpened(new WebWindowEvent(currentWindow_, WebWindowEvent.OPEN, null, null));
257 
258         if (getBrowserVersion().hasFeature(JS_XML_SUPPORT_VIA_ACTIVEXOBJECT)) {
259             initMSXMLActiveX();
260         }
261     }
262 
263     private void initMSXMLActiveX() {
264         msxmlActiveXObjectFactory_ = new MSXMLActiveXObjectFactory();
265         // TODO [IE] initialize in #init or in #initialize?
266         try {
267             msxmlActiveXObjectFactory_.init(getBrowserVersion());
268         }
269         catch (final Exception e) {
270             LOG.error("Exception while initializing MSXML ActiveX for the page", e);
271             throw new ScriptException(null, e); // BUG: null is not useful.
272         }
273     }
274 
275     /**
276      * Returns the object that will resolve all URL requests.
277      *
278      * @return the connection that will be used
279      */
280     public WebConnection getWebConnection() {
281         return webConnection_;
282     }
283 
284     /**
285      * Sets the object that will resolve all URL requests.
286      *
287      * @param webConnection the new web connection
288      */
289     public void setWebConnection(final WebConnection webConnection) {
290         WebAssert.notNull("webConnection", webConnection);
291         webConnection_ = webConnection;
292     }
293 
294     /**
295      * Send a request to a server and return a Page that represents the
296      * response from the server. This page will be used to populate the provided window.
297      * <p>
298      * The returned {@link Page} will be created by the {@link PageCreator}
299      * configured by {@link #setPageCreator(PageCreator)}, if any.
300      * <p>
301      * The {@link DefaultPageCreator} will create a {@link Page} depending on the content type of the HTTP response,
302      * basically {@link HtmlPage} for HTML content, {@link com.gargoylesoftware.htmlunit.xml.XmlPage} for XML content,
303      * {@link TextPage} for other text content and {@link UnexpectedPage} for anything else.
304      *
305      * @param webWindow the WebWindow to load the result of the request into
306      * @param webRequest the web request
307      * @param <P> the page type
308      * @return the page returned by the server when the specified request was made in the specified window
309      * @throws IOException if an IO error occurs
310      * @throws FailingHttpStatusCodeException if the server returns a failing status code AND the property
311      *         {@link WebClientOptions#setThrowExceptionOnFailingStatusCode(boolean)} is set to true
312      *
313      * @see WebRequest
314      */
315     public <P extends Page> P getPage(final WebWindow webWindow, final WebRequest webRequest)
316             throws IOException, FailingHttpStatusCodeException {
317         return getPage(webWindow, webRequest, true);
318     }
319 
320     /**
321      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
322      *
323      * Send a request to a server and return a Page that represents the
324      * response from the server. This page will be used to populate the provided window.
325      * <p>
326      * The returned {@link Page} will be created by the {@link PageCreator}
327      * configured by {@link #setPageCreator(PageCreator)}, if any.
328      * <p>
329      * The {@link DefaultPageCreator} will create a {@link Page} depending on the content type of the HTTP response,
330      * basically {@link HtmlPage} for HTML content, {@link com.gargoylesoftware.htmlunit.xml.XmlPage} for XML content,
331      * {@link TextPage} for other text content and {@link UnexpectedPage} for anything else.
332      *
333      * @param webWindow the WebWindow to load the result of the request into
334      * @param webRequest the web request
335      * @param addToHistory true if the page should be part of the history
336      * @param <P> the page type
337      * @return the page returned by the server when the specified request was made in the specified window
338      * @throws IOException if an IO error occurs
339      * @throws FailingHttpStatusCodeException if the server returns a failing status code AND the property
340      *         {@link WebClientOptions#setThrowExceptionOnFailingStatusCode(boolean)} is set to true
341      *
342      * @see WebRequest
343      */
344     @SuppressWarnings("unchecked")
345     <P extends Page> P getPage(final WebWindow webWindow, final WebRequest webRequest,
346             final boolean addToHistory)
347         throws IOException, FailingHttpStatusCodeException {
348 
349         final Page page = webWindow.getEnclosedPage();
350 
351         if (page != null) {
352             final URL prev = page.getUrl();
353             final URL current = webRequest.getUrl();
354             if (UrlUtils.sameFile(current, prev)
355                         && current.getRef() != null
356                         && !StringUtils.equals(current.getRef(), prev.getRef())) {
357                 // We're just navigating to an anchor within the current page.
358                 page.getWebResponse().getWebRequest().setUrl(current);
359                 if (addToHistory) {
360                     webWindow.getHistory().addPage(page);
361                 }
362 
363                 final Window window = (Window) webWindow.getScriptableObject();
364                 if (window != null) { // js enabled
365                     window.getLocation().setHash(current.getRef());
366                     window.clearComputedStyles();
367                 }
368                 return (P) page;
369             }
370 
371             if (page.isHtmlPage()) {
372                 final HtmlPage htmlPage = (HtmlPage) page;
373                 if (!htmlPage.isOnbeforeunloadAccepted()) {
374                     if (LOG.isDebugEnabled()) {
375                         LOG.debug("The registered OnbeforeunloadHandler rejected to load a new page.");
376                     }
377                     return (P) page;
378                 }
379             }
380         }
381 
382         if (LOG.isDebugEnabled()) {
383             LOG.debug("Get page for window named '" + webWindow.getName() + "', using " + webRequest);
384         }
385 
386         final WebResponse webResponse;
387         final String protocol = webRequest.getUrl().getProtocol();
388         if ("javascript".equals(protocol)) {
389             webResponse = makeWebResponseForJavaScriptUrl(webWindow, webRequest.getUrl(), webRequest.getCharset());
390             if (webWindow.getEnclosedPage() != null && webWindow.getEnclosedPage().getWebResponse() == webResponse) {
391                 // a javascript:... url with result of type undefined didn't changed the page
392                 return (P) webWindow.getEnclosedPage();
393             }
394         }
395         else {
396             webResponse = loadWebResponse(webRequest);
397         }
398 
399         printContentIfNecessary(webResponse);
400         loadWebResponseInto(webResponse, webWindow);
401 
402         // start execution here
403         // note: we have to do this also if the server reports an error!
404         //       e.g. if the server returns a 404 error page that includes javascript
405         if (scriptEngine_ != null) {
406             scriptEngine_.registerWindowAndMaybeStartEventLoop(webWindow);
407         }
408 
409         // check and report problems if needed
410         throwFailingHttpStatusCodeExceptionIfNecessary(webResponse);
411         return (P) webWindow.getEnclosedPage();
412     }
413 
414     /**
415      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
416      *
417      * <p>Open a new web window and populate it with a page loaded by
418      * {@link #getPage(WebWindow,WebRequest)}</p>
419      *
420      * @param opener the web window that initiated the request
421      * @param target the name of the window to be opened (the name that will be passed into the
422      *        JavaScript <tt>open()</tt> method)
423      * @param params any parameters
424      * @param <P> the page type
425      * @return the new page
426      * @throws FailingHttpStatusCodeException if the server returns a failing status code AND the property
427      *         {@link WebClientOptions#setThrowExceptionOnFailingStatusCode(boolean)} is set to true.
428      * @throws IOException if an IO problem occurs
429      */
430     @SuppressWarnings("unchecked")
431     public <P extends Page> P getPage(final WebWindow opener, final String target, final WebRequest params)
432         throws FailingHttpStatusCodeException, IOException {
433         return (P) getPage(openTargetWindow(opener, target, TARGET_SELF), params);
434     }
435 
436     /**
437      * Convenient method to build a URL and load it into the current WebWindow as it would be done
438      * by {@link #getPage(WebWindow, WebRequest)}.
439      * @param url the URL of the new content
440      * @param <P> the page type
441      * @return the new page
442      * @throws FailingHttpStatusCodeException if the server returns a failing status code AND the property
443      *         {@link WebClientOptions#setThrowExceptionOnFailingStatusCode(boolean)} is set to true.
444      * @throws IOException if an IO problem occurs
445      * @throws MalformedURLException if no URL can be created from the provided string
446      */
447     @SuppressWarnings("unchecked")
448     public <P extends Page> P getPage(final String url) throws IOException, FailingHttpStatusCodeException,
449         MalformedURLException {
450         return (P) getPage(UrlUtils.toUrlUnsafe(url));
451     }
452 
453     /**
454      * Convenient method to load a URL into the current top WebWindow as it would be done
455      * by {@link #getPage(WebWindow, WebRequest)}.
456      * @param url the URL of the new content
457      * @param <P> the page type
458      * @return the new page
459      * @throws FailingHttpStatusCodeException if the server returns a failing status code AND the property
460      *         {@link WebClientOptions#setThrowExceptionOnFailingStatusCode(boolean)} is set to true.
461      * @throws IOException if an IO problem occurs
462      */
463     @SuppressWarnings("unchecked")
464     public <P extends Page> P getPage(final URL url) throws IOException, FailingHttpStatusCodeException {
465         return (P) getPage(getCurrentWindow().getTopWindow(),
466                 new WebRequest(url, getBrowserVersion().getHtmlAcceptHeader()));
467     }
468 
469     /**
470      * Convenient method to load a web request into the current top WebWindow.
471      * @param request the request parameters
472      * @param <P> the page type
473      * @return the new page
474      * @throws FailingHttpStatusCodeException if the server returns a failing status code AND the property
475      *         {@link WebClientOptions#setThrowExceptionOnFailingStatusCode(boolean)} is set to true.
476      * @throws IOException if an IO problem occurs
477      * @see #getPage(WebWindow,WebRequest)
478      */
479     @SuppressWarnings("unchecked")
480     public <P extends Page> P getPage(final WebRequest request) throws IOException,
481         FailingHttpStatusCodeException {
482         return (P) getPage(getCurrentWindow().getTopWindow(), request);
483     }
484 
485     /**
486      * <p>Creates a page based on the specified response and inserts it into the specified window. All page
487      * initialization and event notification is handled here.</p>
488      *
489      * <p>Note that if the page created is an attachment page, and an {@link AttachmentHandler} has been
490      * registered with this client, the page is <b>not</b> loaded into the specified window; in this case,
491      * the page is loaded into a new window, and attachment handling is delegated to the registered
492      * <tt>AttachmentHandler</tt>.</p>
493      *
494      * @param webResponse the response that will be used to create the new page
495      * @param webWindow the window that the new page will be placed within
496      * @throws IOException if an IO error occurs
497      * @throws FailingHttpStatusCodeException if the server returns a failing status code AND the property
498      *         {@link WebClientOptions#setThrowExceptionOnFailingStatusCode(boolean)} is set to true
499      * @return the newly created page
500      * @see #setAttachmentHandler(AttachmentHandler)
501      */
502     public Page loadWebResponseInto(final WebResponse webResponse, final WebWindow webWindow)
503         throws IOException, FailingHttpStatusCodeException {
504 
505         WebAssert.notNull("webResponse", webResponse);
506         WebAssert.notNull("webWindow", webWindow);
507 
508         if (webResponse.getStatusCode() == HttpStatus.SC_NO_CONTENT) {
509             return webWindow.getEnclosedPage();
510         }
511 
512         if (webStartHandler_ != null && "application/x-java-jnlp-file".equals(webResponse.getContentType())) {
513             webStartHandler_.handleJnlpResponse(webResponse);
514             return webWindow.getEnclosedPage();
515         }
516 
517         if (attachmentHandler_ != null && Attachment.isAttachment(webResponse)) {
518             final WebWindow w = openWindow(null, null, webWindow);
519             final Page page = pageCreator_.createPage(webResponse, w);
520             attachmentHandler_.handleAttachment(page);
521             return page;
522         }
523 
524         final Page oldPage = webWindow.getEnclosedPage();
525         if (oldPage != null) {
526             // Remove the old windows before create new ones.
527             oldPage.cleanUp();
528         }
529         Page newPage = null;
530         if (windows_.contains(webWindow) || getBrowserVersion().hasFeature(WINDOW_EXECUTE_EVENTS)) {
531             newPage = pageCreator_.createPage(webResponse, webWindow);
532 
533             if (windows_.contains(webWindow)) {
534                 fireWindowContentChanged(new WebWindowEvent(webWindow, WebWindowEvent.CHANGE, oldPage, newPage));
535 
536                 // The page being loaded may already have been replaced by another page via JavaScript code.
537                 if (webWindow.getEnclosedPage() == newPage) {
538                     newPage.initialize();
539                     // hack: onload should be fired the same way for all type of pages
540                     // here is a hack to handle non HTML pages
541                     if (webWindow instanceof FrameWindow && !newPage.isHtmlPage()) {
542                         final FrameWindow fw = (FrameWindow) webWindow;
543                         final BaseFrameElement frame = fw.getFrameElement();
544                         if (frame.hasEventHandlers("onload")) {
545                             if (LOG.isDebugEnabled()) {
546                                 LOG.debug("Executing onload handler for " + frame);
547                             }
548                             final Event event = new Event(frame, Event.TYPE_LOAD);
549                             ((Node) frame.getScriptableObject()).executeEventLocally(event);
550                         }
551                     }
552                 }
553             }
554         }
555         return newPage;
556     }
557 
558     /**
559      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span>
560      *
561      * <p>Logs the response's content if its status code indicates a request failure and
562      * {@link WebClientOptions#isPrintContentOnFailingStatusCode()} returns {@code true}.
563      *
564      * @param webResponse the response whose content may be logged
565      */
566     public void printContentIfNecessary(final WebResponse webResponse) {
567         if (getOptions().isPrintContentOnFailingStatusCode()) {
568             final int statusCode = webResponse.getStatusCode();
569             final boolean successful = statusCode >= HttpStatus.SC_OK && statusCode < HttpStatus.SC_MULTIPLE_CHOICES;
570             if (!successful) {
571                 final String contentType = webResponse.getContentType();
572                 LOG.info("statusCode=[" + statusCode + "] contentType=[" + contentType + "]");
573                 LOG.info(webResponse.getContentAsString());
574             }
575         }
576     }
577 
578     /**
579      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span>
580      *
581      * <p>Throws a {@link FailingHttpStatusCodeException} if the request's status code indicates a request
582      * failure and {@link WebClientOptions#isThrowExceptionOnFailingStatusCode()} returns {@code true}.
583      *
584      * @param webResponse the response which may trigger a {@link FailingHttpStatusCodeException}
585      */
586     public void throwFailingHttpStatusCodeExceptionIfNecessary(final WebResponse webResponse) {
587         final int statusCode = webResponse.getStatusCode();
588         final boolean successful = (statusCode >= HttpStatus.SC_OK && statusCode < HttpStatus.SC_MULTIPLE_CHOICES)
589             || statusCode == HttpStatus.SC_USE_PROXY
590             || statusCode == HttpStatus.SC_NOT_MODIFIED;
591         if (getOptions().isThrowExceptionOnFailingStatusCode() && !successful) {
592             throw new FailingHttpStatusCodeException(webResponse);
593         }
594     }
595 
596     /**
597      * Adds a header which will be sent with EVERY request from this client.
598      * @param name the name of the header to add
599      * @param value the value of the header to add
600      * @see #removeRequestHeader(String)
601      */
602     public void addRequestHeader(final String name, final String value) {
603         if (HttpHeader.COOKIE_LC.equalsIgnoreCase(name)) {
604             throw new IllegalArgumentException("Do not add 'Cookie' header, use .getCookieManager() instead");
605         }
606         requestHeaders_.put(name, value);
607     }
608 
609     /**
610      * Removes a header from being sent with EVERY request from this client.
611      * @param name the name of the header to remove
612      * @see #addRequestHeader
613      */
614     public void removeRequestHeader(final String name) {
615         requestHeaders_.remove(name);
616     }
617 
618     /**
619      * Sets the credentials provider that will provide authentication information when
620      * trying to access protected information on a web server. This information is
621      * required when the server is using Basic HTTP authentication, NTLM authentication,
622      * or Digest authentication.
623      * @param credentialsProvider the new credentials provider to use to authenticate
624      */
625     public void setCredentialsProvider(final CredentialsProvider credentialsProvider) {
626         WebAssert.notNull("credentialsProvider", credentialsProvider);
627         credentialsProvider_ = credentialsProvider;
628     }
629 
630     /**
631      * Returns the credentials provider for this client instance. By default, this
632      * method returns an instance of {@link DefaultCredentialsProvider}.
633      * @return the credentials provider for this client instance
634      */
635     public CredentialsProvider getCredentialsProvider() {
636         return credentialsProvider_;
637     }
638 
639     /**
640      * This method is intended for testing only - use at your own risk.
641      * @return the current JavaScript engine (never {@code null})
642      */
643     public AbstractJavaScriptEngine<?> getJavaScriptEngine() {
644         return scriptEngine_;
645     }
646 
647     /**
648      * This method is intended for testing only - use at your own risk.
649      *
650      * @param engine the new script engine to use
651      */
652     public void setJavaScriptEngine(final AbstractJavaScriptEngine<?> engine) {
653         if (engine == null) {
654             throw new IllegalArgumentException("Can't set JavaScriptEngine to null");
655         }
656         scriptEngine_ = engine;
657     }
658 
659     /**
660      * Returns the cookie manager used by this web client.
661      * @return the cookie manager used by this web client
662      */
663     public CookieManager getCookieManager() {
664         return cookieManager_;
665     }
666 
667     /**
668      * Sets the cookie manager used by this web client.
669      * @param cookieManager the cookie manager used by this web client
670      */
671     public void setCookieManager(final CookieManager cookieManager) {
672         WebAssert.notNull("cookieManager", cookieManager);
673         cookieManager_ = cookieManager;
674     }
675 
676     /**
677      * Sets the alert handler for this webclient.
678      * @param alertHandler the new alerthandler or null if none is specified
679      */
680     public void setAlertHandler(final AlertHandler alertHandler) {
681         alertHandler_ = alertHandler;
682     }
683 
684     /**
685      * Returns the alert handler for this webclient.
686      * @return the alert handler or null if one hasn't been set
687      */
688     public AlertHandler getAlertHandler() {
689         return alertHandler_;
690     }
691 
692     /**
693      * Sets the handler that will be executed when the JavaScript method Window.confirm() is called.
694      * @param handler the new handler or null if no handler is to be used
695      */
696     public void setConfirmHandler(final ConfirmHandler handler) {
697         confirmHandler_ = handler;
698     }
699 
700     /**
701      * Returns the confirm handler.
702      * @return the confirm handler or null if one hasn't been set
703      */
704     public ConfirmHandler getConfirmHandler() {
705         return confirmHandler_;
706     }
707 
708     /**
709      * Sets the handler that will be executed when the JavaScript method Window.prompt() is called.
710      * @param handler the new handler or null if no handler is to be used
711      */
712     public void setPromptHandler(final PromptHandler handler) {
713         promptHandler_ = handler;
714     }
715 
716     /**
717      * Returns the prompt handler.
718      * @return the prompt handler or null if one hasn't been set
719      */
720     public PromptHandler getPromptHandler() {
721         return promptHandler_;
722     }
723 
724     /**
725      * Sets the status handler for this webclient.
726      * @param statusHandler the new status handler or null if none is specified
727      */
728     public void setStatusHandler(final StatusHandler statusHandler) {
729         statusHandler_ = statusHandler;
730     }
731 
732     /**
733      * Returns the status handler for this webclient.
734      * @return the status handler or null if one hasn't been set
735      */
736     public StatusHandler getStatusHandler() {
737         return statusHandler_;
738     }
739 
740     /**
741      * Sets the javascript error listener for this webclient.
742      * When setting to null, the {@link DefaultJavaScriptErrorListener} is used.
743      * @param javaScriptErrorListener the new JavaScriptErrorListener or null if none is specified
744      */
745     public void setJavaScriptErrorListener(final JavaScriptErrorListener javaScriptErrorListener) {
746         if (javaScriptErrorListener == null) {
747             javaScriptErrorListener_ = new DefaultJavaScriptErrorListener();
748         }
749         else {
750             javaScriptErrorListener_ = javaScriptErrorListener;
751         }
752     }
753 
754     /**
755      * Returns the javascript error listener for this webclient.
756      * @return the javascript error listener or null if one hasn't been set
757      */
758     public JavaScriptErrorListener getJavaScriptErrorListener() {
759         return javaScriptErrorListener_;
760     }
761 
762     /**
763      * Returns the current browser version.
764      * @return the current browser version
765      */
766     public BrowserVersion getBrowserVersion() {
767         return browserVersion_;
768     }
769 
770     /**
771      * Returns the "current" window for this client. This window (or its top window) will be used
772      * when <tt>getPage(...)</tt> is called without specifying a window.
773      * @return the "current" window for this client
774      */
775     public WebWindow getCurrentWindow() {
776         return currentWindow_;
777     }
778 
779     /**
780      * Sets the "current" window for this client. This is the window that will be used when
781      * <tt>getPage(...)</tt> is called without specifying a window.
782      * @param window the new "current" window for this client
783      */
784     public void setCurrentWindow(final WebWindow window) {
785         WebAssert.notNull("window", window);
786         if (currentWindow_ == window) {
787             return;
788         }
789         // onBlur event is triggered for focused element of old current window
790         if (currentWindow_ != null && !currentWindow_.isClosed()) {
791             final Page enclosedPage = currentWindow_.getEnclosedPage();
792             if (enclosedPage != null && enclosedPage.isHtmlPage()) {
793                 final DomElement focusedElement = ((HtmlPage) enclosedPage).getFocusedElement();
794                 if (focusedElement != null) {
795                     focusedElement.fireEvent(Event.TYPE_BLUR);
796                 }
797             }
798         }
799         currentWindow_ = window;
800 
801         // when marking an iframe window as current we have no need to move the focus
802         final boolean isIFrame = currentWindow_ instanceof FrameWindow
803                 && ((FrameWindow) currentWindow_).getFrameElement() instanceof HtmlInlineFrame;
804         if (!isIFrame) {
805             //1. activeElement becomes focused element for new current window
806             //2. onFocus event is triggered for focusedElement of new current window
807             final Page enclosedPage = currentWindow_.getEnclosedPage();
808             if (enclosedPage != null && enclosedPage.isHtmlPage()) {
809                 final Object jsWindow = currentWindow_.getScriptableObject();
810                 if (jsWindow instanceof Window) {
811                     final HTMLElement activeElement =
812                             ((HTMLDocument) ((Window) jsWindow).getDocument()).getActiveElement();
813                     if (activeElement != null) {
814                         ((HtmlPage) enclosedPage).setFocusedElement(activeElement.getDomNodeOrDie(), true);
815                     }
816                 }
817             }
818         }
819     }
820 
821     /**
822      * Adds a listener for {@link WebWindowEvent}s. All events from all windows associated with this
823      * client will be sent to the specified listener.
824      * @param listener a listener
825      */
826     public void addWebWindowListener(final WebWindowListener listener) {
827         WebAssert.notNull("listener", listener);
828         webWindowListeners_.add(listener);
829     }
830 
831     /**
832      * Removes a listener for {@link WebWindowEvent}s.
833      * @param listener a listener
834      */
835     public void removeWebWindowListener(final WebWindowListener listener) {
836         WebAssert.notNull("listener", listener);
837         webWindowListeners_.remove(listener);
838     }
839 
840     private void fireWindowContentChanged(final WebWindowEvent event) {
841         for (final WebWindowListener listener : new ArrayList<>(webWindowListeners_)) {
842             listener.webWindowContentChanged(event);
843         }
844     }
845 
846     private void fireWindowOpened(final WebWindowEvent event) {
847         for (final WebWindowListener listener : new ArrayList<>(webWindowListeners_)) {
848             listener.webWindowOpened(event);
849         }
850     }
851 
852     private void fireWindowClosed(final WebWindowEvent event) {
853         for (final WebWindowListener listener : new ArrayList<>(webWindowListeners_)) {
854             listener.webWindowClosed(event);
855         }
856     }
857 
858     /**
859      * Open a new window with the specified name. If the URL is non-null then attempt to load
860      * a page from that location and put it in the new window.
861      *
862      * @param url the URL to load content from or null if no content is to be loaded
863      * @param windowName the name of the new window
864      * @return the new window
865      */
866     public WebWindow openWindow(final URL url, final String windowName) {
867         WebAssert.notNull("windowName", windowName);
868         return openWindow(url, windowName, getCurrentWindow());
869     }
870 
871     /**
872      * Open a new window with the specified name. If the URL is non-null then attempt to load
873      * a page from that location and put it in the new window.
874      *
875      * @param url the URL to load content from or null if no content is to be loaded
876      * @param windowName the name of the new window
877      * @param opener the web window that is calling openWindow
878      * @return the new window
879      */
880     public WebWindow openWindow(final URL url, final String windowName, final WebWindow opener) {
881         final WebWindow window = openTargetWindow(opener, windowName, TARGET_BLANK);
882         final HtmlPage openerPage = (HtmlPage) opener.getEnclosedPage();
883         if (url != null) {
884             try {
885                 final WebRequest request = new WebRequest(url, getBrowserVersion().getHtmlAcceptHeader());
886                 if (getBrowserVersion().hasFeature(DIALOGWINDOW_REFERER)
887                         && openerPage != null) {
888                     final String referer = openerPage.getUrl().toExternalForm();
889                     request.setAdditionalHeader(HttpHeader.REFERER, referer);
890                 }
891                 getPage(window, request);
892             }
893             catch (final IOException e) {
894                 LOG.error("Error loading content into window", e);
895             }
896         }
897         else {
898             initializeEmptyWindow(window);
899         }
900         return window;
901     }
902 
903     /**
904      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
905      *
906      * Open the window with the specified name. The name may be a special
907      * target name of _self, _parent, _top, or _blank. An empty or null
908      * name is set to the default. The special target names are relative to
909      * the opener window.
910      *
911      * @param opener the web window that is calling openWindow
912      * @param windowName the name of the new window
913      * @param defaultName the default target if no name is given
914      * @return the new window
915      */
916     public WebWindow openTargetWindow(
917             final WebWindow opener, final String windowName, final String defaultName) {
918 
919         WebAssert.notNull("opener", opener);
920         WebAssert.notNull("defaultName", defaultName);
921 
922         String windowToOpen = windowName;
923         if (windowToOpen == null || windowToOpen.isEmpty()) {
924             windowToOpen = defaultName;
925         }
926 
927         WebWindow webWindow = resolveWindow(opener, windowToOpen);
928 
929         if (webWindow == null) {
930             if (TARGET_BLANK.equals(windowToOpen)) {
931                 windowToOpen = "";
932             }
933             webWindow = new TopLevelWindow(windowToOpen, this);
934             fireWindowOpened(new WebWindowEvent(webWindow, WebWindowEvent.OPEN, null, null));
935         }
936 
937         if (webWindow instanceof TopLevelWindow && webWindow != opener.getTopWindow()) {
938             ((TopLevelWindow) webWindow).setOpener(opener);
939         }
940 
941         return webWindow;
942     }
943 
944     private WebWindow resolveWindow(final WebWindow opener, final String name) {
945         if (name == null || name.isEmpty() || TARGET_SELF.equals(name)) {
946             return opener;
947         }
948 
949         if (TARGET_PARENT.equals(name)) {
950             return opener.getParentWindow();
951         }
952 
953         if (TARGET_TOP.equals(name)) {
954             return opener.getTopWindow();
955         }
956 
957         if (TARGET_BLANK.equals(name)) {
958             return null;
959         }
960 
961         // first search for frame windows inside our window hierarchy
962         WebWindow window = opener;
963         while (true) {
964             final Page page = window.getEnclosedPage();
965             if (page != null && page.isHtmlPage()) {
966                 try {
967                     final FrameWindow frame = ((HtmlPage) page).getFrameByName(name);
968                     final ScriptableObject scriptable = frame.getFrameElement().getScriptableObject();
969                     if (scriptable instanceof HTMLIFrameElement) {
970                         ((HTMLIFrameElement) scriptable).onRefresh();
971                     }
972                     return frame;
973                 }
974                 catch (final ElementNotFoundException e) {
975                     // Fall through
976                 }
977             }
978 
979             if (window == window.getParentWindow()) {
980                 // TODO: should getParentWindow() return null on top windows?
981                 break;
982             }
983             window = window.getParentWindow();
984         }
985 
986         try {
987             return getWebWindowByName(name);
988         }
989         catch (final WebWindowNotFoundException e) {
990             // Fall through - a new window will be created below
991         }
992         return null;
993     }
994 
995     /**
996      * <p><span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span></p>
997      *
998      * Opens a new dialog window.
999      * @param url the URL of the document to load and display
1000      * @param opener the web window that is opening the dialog
1001      * @param dialogArguments the object to make available inside the dialog via <tt>window.dialogArguments</tt>
1002      * @return the new dialog window
1003      * @throws IOException if there is an IO error
1004      */
1005     public DialogWindow openDialogWindow(final URL url, final WebWindow opener, final Object dialogArguments)
1006         throws IOException {
1007 
1008         WebAssert.notNull("url", url);
1009         WebAssert.notNull("opener", opener);
1010 
1011         final DialogWindow window = new DialogWindow(this, dialogArguments);
1012         fireWindowOpened(new WebWindowEvent(window, WebWindowEvent.OPEN, null, null));
1013 
1014         final HtmlPage openerPage = (HtmlPage) opener.getEnclosedPage();
1015         final WebRequest request = new WebRequest(url, getBrowserVersion().getHtmlAcceptHeader());
1016         if (getBrowserVersion().hasFeature(DIALOGWINDOW_REFERER) && openerPage != null) {
1017             final String referer = openerPage.getUrl().toExternalForm();
1018             request.setAdditionalHeader(HttpHeader.REFERER, referer);
1019         }
1020 
1021         getPage(window, request);
1022 
1023         return window;
1024     }
1025 
1026     /**
1027      * Sets the object that will be used to create pages. Set this if you want
1028      * to customize the type of page that is returned for a given content type.
1029      *
1030      * @param pageCreator the new page creator
1031      */
1032     public void setPageCreator(final PageCreator pageCreator) {
1033         WebAssert.notNull("pageCreator", pageCreator);
1034         pageCreator_ = pageCreator;
1035     }
1036 
1037     /**
1038      * Returns the current page creator.
1039      *
1040      * @return the page creator
1041      */
1042     public PageCreator getPageCreator() {
1043         return pageCreator_;
1044     }
1045 
1046     /**
1047      * Returns the first {@link WebWindow} that matches the specified name.
1048      *
1049      * @param name the name to search for
1050      * @return the {@link WebWindow} with the specified name
1051      * @throws WebWindowNotFoundException if the {@link WebWindow} can't be found
1052      * @see #getWebWindows()
1053      * @see #getTopLevelWindows()
1054      */
1055     public WebWindow getWebWindowByName(final String name) throws WebWindowNotFoundException {
1056         WebAssert.notNull("name", name);
1057 
1058         for (final WebWindow webWindow : windows_) {
1059             if (name.equals(webWindow.getName())) {
1060                 return webWindow;
1061             }
1062         }
1063 
1064         throw new WebWindowNotFoundException(name);
1065     }
1066 
1067     /**
1068      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
1069      *
1070      * Initializes a new web window for JavaScript.
1071      * @param webWindow the new WebWindow
1072      */
1073     public void initialize(final WebWindow webWindow) {
1074         WebAssert.notNull("webWindow", webWindow);
1075         scriptEngine_.initialize(webWindow);
1076     }
1077 
1078     /**
1079      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
1080      *
1081      * Initializes a new page for JavaScript.
1082      * @param newPage the new page
1083      */
1084     public void initialize(final Page newPage) {
1085         WebAssert.notNull("newPage", newPage);
1086         final WebWindow webWindow = newPage.getEnclosingWindow();
1087         if (webWindow.getScriptableObject() instanceof Window) {
1088             ((Window) webWindow.getScriptableObject()).initialize(newPage);
1089         }
1090     }
1091 
1092     /**
1093      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
1094      *
1095      * Initializes a new empty window for JavaScript.
1096      *
1097      * @param webWindow the new WebWindow
1098      */
1099     public void initializeEmptyWindow(final WebWindow webWindow) {
1100         WebAssert.notNull("webWindow", webWindow);
1101         initialize(webWindow);
1102         ((Window) webWindow.getScriptableObject()).initialize();
1103     }
1104 
1105     /**
1106      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
1107      *
1108      * Adds a new window to the list of available windows.
1109      *
1110      * @param webWindow the new WebWindow
1111      */
1112     public void registerWebWindow(final WebWindow webWindow) {
1113         WebAssert.notNull("webWindow", webWindow);
1114         windows_.add(webWindow);
1115         // register JobManager here but don't deregister in deregisterWebWindow as it can live longer
1116         jobManagers_.add(new WeakReference<>(webWindow.getJobManager()));
1117     }
1118 
1119     /**
1120      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
1121      *
1122      * Removes a window from the list of available windows.
1123      *
1124      * @param webWindow the window to remove
1125      */
1126     public void deregisterWebWindow(final WebWindow webWindow) {
1127         WebAssert.notNull("webWindow", webWindow);
1128         if (windows_.remove(webWindow)) {
1129             fireWindowClosed(new WebWindowEvent(webWindow, WebWindowEvent.CLOSE, webWindow.getEnclosedPage(), null));
1130         }
1131     }
1132 
1133     /**
1134      * Expands a relative URL relative to the specified base. In most situations
1135      * this is the same as <code>new URL(baseUrl, relativeUrl)</code> but
1136      * there are some cases that URL doesn't handle correctly. See
1137      * <a href="http://www.faqs.org/rfcs/rfc1808.html">RFC1808</a>
1138      * regarding Relative Uniform Resource Locators for more information.
1139      *
1140      * @param baseUrl the base URL
1141      * @param relativeUrl the relative URL
1142      * @return the expansion of the specified base and relative URLs
1143      * @throws MalformedURLException if an error occurred when creating a URL object
1144      */
1145     public static URL expandUrl(final URL baseUrl, final String relativeUrl) throws MalformedURLException {
1146         final String newUrl = UrlUtils.resolveUrl(baseUrl, relativeUrl);
1147         return UrlUtils.toUrlUnsafe(newUrl);
1148     }
1149 
1150     private WebResponse makeWebResponseForDataUrl(final WebRequest webRequest) throws IOException {
1151         final URL url = webRequest.getUrl();
1152         final List<NameValuePair> responseHeaders = new ArrayList<>();
1153         final DataURLConnection connection;
1154         try {
1155             connection = new DataURLConnection(url);
1156         }
1157         catch (final DecoderException e) {
1158             throw new IOException(e.getMessage());
1159         }
1160         responseHeaders.add(new NameValuePair(HttpHeader.CONTENT_TYPE_LC,
1161             connection.getMediaType() + ";charset=" + connection.getCharset()));
1162 
1163         try (InputStream is = connection.getInputStream()) {
1164             final DownloadedContent downloadedContent =
1165                     HttpWebConnection.downloadContent(is, getOptions().getMaxInMemory());
1166             final WebResponseData data = new WebResponseData(downloadedContent, 200, "OK", responseHeaders);
1167             return new WebResponse(data, url, webRequest.getHttpMethod(), 0);
1168         }
1169     }
1170 
1171     private static WebResponse makeWebResponseForAboutUrl(final URL url) {
1172         final String urlWithoutQuery = StringUtils.substringBefore(url.toExternalForm(), "?");
1173         if (!"blank".equalsIgnoreCase(StringUtils.substringAfter(urlWithoutQuery, WebClient.ABOUT_SCHEME))) {
1174             throw new IllegalArgumentException(url + " is not supported, only about:blank is supported now.");
1175         }
1176         return new StringWebResponse("", URL_ABOUT_BLANK);
1177     }
1178 
1179     /**
1180      * Builds a WebResponse for a file URL.
1181      * This first implementation is basic.
1182      * It assumes that the file contains an HTML page encoded with the specified encoding.
1183      * @param url the file URL
1184      * @param charset encoding to use
1185      * @return the web response
1186      * @throws IOException if an IO problem occurs
1187      */
1188     private WebResponse makeWebResponseForFileUrl(final WebRequest webRequest) throws IOException {
1189         URL cleanUrl = webRequest.getUrl();
1190         if (cleanUrl.getQuery() != null) {
1191             // Get rid of the query portion before trying to load the file.
1192             cleanUrl = UrlUtils.getUrlWithNewQuery(cleanUrl, null);
1193         }
1194         if (cleanUrl.getRef() != null) {
1195             // Get rid of the ref portion before trying to load the file.
1196             cleanUrl = UrlUtils.getUrlWithNewRef(cleanUrl, null);
1197         }
1198 
1199         String fileUrl = cleanUrl.toExternalForm();
1200         fileUrl = URLDecoder.decode(fileUrl, UTF_8.name());
1201         final File file = new File(fileUrl.substring(5));
1202         if (!file.exists()) {
1203             // construct 404
1204             final List<NameValuePair> compiledHeaders = new ArrayList<>();
1205             compiledHeaders.add(new NameValuePair(HttpHeader.CONTENT_TYPE, "text/html"));
1206             final WebResponseData responseData =
1207                 new WebResponseData(
1208                     TextUtil.stringToByteArray("File: " + file.getAbsolutePath(), UTF_8),
1209                     404, "Not Found", compiledHeaders);
1210             return new WebResponse(responseData, webRequest, 0);
1211         }
1212 
1213         final String contentType = guessContentType(file);
1214 
1215         final DownloadedContent content = new DownloadedContent.OnFile(file, false);
1216         final List<NameValuePair> compiledHeaders = new ArrayList<>();
1217         compiledHeaders.add(new NameValuePair(HttpHeader.CONTENT_TYPE, contentType));
1218         final WebResponseData responseData = new WebResponseData(content, 200, "OK", compiledHeaders);
1219         return new WebResponse(responseData, webRequest, 0);
1220     }
1221 
1222     /**
1223      * Tries to guess the content type of the file.<br>
1224      * This utility could be located in a helper class but we can compare this functionality
1225      * for instance with the "Helper Applications" settings of Mozilla and therefore see it as a
1226      * property of the "browser".
1227      * @param file the file
1228      * @return "application/octet-stream" if nothing could be guessed
1229      */
1230     public String guessContentType(final File file) {
1231         final String fileName = file.getName();
1232         if (fileName.endsWith(".xhtml")) {
1233             // Java's mime type map returns application/xml in JDK8.
1234             return "application/xhtml+xml";
1235         }
1236 
1237         // Java's mime type map does not know these in JDK8.
1238         if (fileName.endsWith(".js")) {
1239             return "text/javascript";
1240         }
1241         if (fileName.toLowerCase(Locale.ROOT).endsWith(".css")) {
1242             return "text/css";
1243         }
1244 
1245         String contentType = URLConnection.guessContentTypeFromName(fileName);
1246         if (contentType == null) {
1247             try (InputStream inputStream = new BufferedInputStream(new FileInputStream(file))) {
1248                 contentType = URLConnection.guessContentTypeFromStream(inputStream);
1249             }
1250             catch (final IOException e) {
1251                 // Ignore silently.
1252             }
1253         }
1254         if (contentType == null) {
1255             contentType = "application/octet-stream";
1256         }
1257         return contentType;
1258     }
1259 
1260     private WebResponse makeWebResponseForJavaScriptUrl(final WebWindow webWindow, final URL url,
1261         final Charset charset) throws FailingHttpStatusCodeException, IOException {
1262 
1263         HtmlPage page = null;
1264         if (webWindow instanceof FrameWindow) {
1265             final FrameWindow frameWindow = (FrameWindow) webWindow;
1266             page = (HtmlPage) frameWindow.getEnclosedPage();
1267         }
1268         else {
1269             Page currentPage = webWindow.getEnclosedPage();
1270             if (currentPage == null) {
1271                 // Starting with a JavaScript URL; quickly fill an "about:blank".
1272                 currentPage = getPage(webWindow, new WebRequest(WebClient.URL_ABOUT_BLANK));
1273             }
1274             else if (currentPage instanceof HtmlPage) {
1275                 page = (HtmlPage) currentPage;
1276             }
1277         }
1278 
1279         if (page == null) {
1280             page = getPage(webWindow, new WebRequest(WebClient.URL_ABOUT_BLANK));
1281         }
1282         final ScriptResult r = page.executeJavaScript(url.toExternalForm(), "JavaScript URL", 1);
1283         if (r.getJavaScriptResult() == null || ScriptResult.isUndefined(r)) {
1284             // No new WebResponse to produce.
1285             return webWindow.getEnclosedPage().getWebResponse();
1286         }
1287 
1288         final String contentString = r.getJavaScriptResult().toString();
1289         final StringWebResponse response = new StringWebResponse(contentString, charset, url);
1290         response.setFromJavascript(true);
1291         return response;
1292     }
1293 
1294     /**
1295      * Loads a {@link WebResponse} from the server.
1296      * @param webRequest the request
1297      * @throws IOException if an IO problem occurs
1298      * @return the WebResponse
1299      */
1300     public WebResponse loadWebResponse(final WebRequest webRequest) throws IOException {
1301         switch (webRequest.getUrl().getProtocol()) {
1302             case "about":
1303                 return makeWebResponseForAboutUrl(webRequest.getUrl());
1304 
1305             case "file":
1306                 return makeWebResponseForFileUrl(webRequest);
1307 
1308             case "data":
1309                 return makeWebResponseForDataUrl(webRequest);
1310 
1311             default:
1312                 return loadWebResponseFromWebConnection(webRequest, ALLOWED_REDIRECTIONS_SAME_URL,
1313                                                             webRequest.getCharset());
1314         }
1315     }
1316 
1317     /**
1318      * Loads a {@link WebResponse} from the server through the WebConnection.
1319      * @param webRequest the request
1320      * @param allowedRedirects the number of allowed redirects remaining
1321      * @param charset the charset to use
1322      * @throws IOException if an IO problem occurs
1323      * @return the resultant {@link WebResponse}
1324      */
1325     private WebResponse loadWebResponseFromWebConnection(final WebRequest webRequest,
1326         final int allowedRedirects, final Charset charset) throws IOException {
1327 
1328         URL url = webRequest.getUrl();
1329         final HttpMethod method = webRequest.getHttpMethod();
1330         final List<NameValuePair> parameters = webRequest.getRequestParameters();
1331 
1332         WebAssert.notNull("url", url);
1333         WebAssert.notNull("method", method);
1334         WebAssert.notNull("parameters", parameters);
1335 
1336         url = UrlUtils.encodeUrl(url, getBrowserVersion().hasFeature(URL_MINIMAL_QUERY_ENCODING), charset);
1337         webRequest.setUrl(url);
1338 
1339         if (LOG.isDebugEnabled()) {
1340             LOG.debug("Load response for " + method + " " + url.toExternalForm());
1341         }
1342 
1343         // If the request settings don't specify a custom proxy, use the default client proxy...
1344         if (webRequest.getProxyHost() == null) {
1345             final ProxyConfig proxyConfig = getOptions().getProxyConfig();
1346             if (proxyConfig.getProxyAutoConfigUrl() != null) {
1347                 if (!UrlUtils.sameFile(new URL(proxyConfig.getProxyAutoConfigUrl()), url)) {
1348                     String content = proxyConfig.getProxyAutoConfigContent();
1349                     if (content == null) {
1350                         content = getPage(proxyConfig.getProxyAutoConfigUrl())
1351                             .getWebResponse().getContentAsString();
1352                         proxyConfig.setProxyAutoConfigContent(content);
1353                     }
1354                     final String allValue = ProxyAutoConfig.evaluate(content, url);
1355                     if (LOG.isDebugEnabled()) {
1356                         LOG.debug("Proxy Auto-Config: value '" + allValue + "' for URL " + url);
1357                     }
1358                     String value = allValue.split(";")[0].trim();
1359                     if (value.startsWith("PROXY")) {
1360                         value = value.substring(6);
1361                         final int colonIndex = value.indexOf(':');
1362                         webRequest.setSocksProxy(false);
1363                         webRequest.setProxyHost(value.substring(0, colonIndex));
1364                         webRequest.setProxyPort(Integer.parseInt(value.substring(colonIndex + 1)));
1365                     }
1366                     else if (value.startsWith("SOCKS")) {
1367                         value = value.substring(6);
1368                         final int colonIndex = value.indexOf(':');
1369                         webRequest.setSocksProxy(true);
1370                         webRequest.setProxyHost(value.substring(0, colonIndex));
1371                         webRequest.setProxyPort(Integer.parseInt(value.substring(colonIndex + 1)));
1372                     }
1373                 }
1374             }
1375             // ...unless the host needs to bypass the configured client proxy!
1376             else if (!proxyConfig.shouldBypassProxy(webRequest.getUrl().getHost())) {
1377                 webRequest.setProxyHost(proxyConfig.getProxyHost());
1378                 webRequest.setProxyPort(proxyConfig.getProxyPort());
1379                 webRequest.setSocksProxy(proxyConfig.isSocksProxy());
1380             }
1381         }
1382 
1383         // Add the headers that are sent with every request.
1384         addDefaultHeaders(webRequest);
1385 
1386         // Retrieve the response, either from the cache or from the server.
1387         final WebResponse fromCache = getCache().getCachedResponse(webRequest);
1388         final WebResponse webResponse;
1389         if (fromCache != null) {
1390             webResponse = new WebResponseFromCache(fromCache, webRequest);
1391         }
1392         else {
1393             try {
1394                 webResponse = getWebConnection().getResponse(webRequest);
1395             }
1396             catch (final NoHttpResponseException e) {
1397                 return new WebResponse(responseDataNoHttpResponse_, webRequest, 0);
1398             }
1399             getCache().cacheIfPossible(webRequest, webResponse, null);
1400         }
1401 
1402         // Continue according to the HTTP status code.
1403         final int status = webResponse.getStatusCode();
1404         if (status == HttpStatus.SC_USE_PROXY) {
1405             getIncorrectnessListener().notify("Ignoring HTTP status code [305] 'Use Proxy'", this);
1406         }
1407         else if (status >= HttpStatus.SC_MOVED_PERMANENTLY
1408             && status <= (getBrowserVersion().hasFeature(HTTP_REDIRECT_308) ? 308 : HttpStatus.SC_TEMPORARY_REDIRECT)
1409             && status != HttpStatus.SC_NOT_MODIFIED
1410             && getOptions().isRedirectEnabled()) {
1411 
1412             final URL newUrl;
1413             String locationString = null;
1414             try {
1415                 locationString = webResponse.getResponseHeaderValue("Location");
1416                 if (locationString == null) {
1417                     return webResponse;
1418                 }
1419                 if (!getBrowserVersion().hasFeature(URL_MINIMAL_QUERY_ENCODING)) {
1420                     locationString = new String(locationString.getBytes(ISO_8859_1), UTF_8);
1421                 }
1422                 newUrl = expandUrl(url, locationString);
1423             }
1424             catch (final MalformedURLException e) {
1425                 getIncorrectnessListener().notify("Got a redirect status code [" + status + " "
1426                     + webResponse.getStatusMessage()
1427                     + "] but the location is not a valid URL [" + locationString
1428                     + "]. Skipping redirection processing.", this);
1429                 return webResponse;
1430             }
1431 
1432             if (LOG.isDebugEnabled()) {
1433                 LOG.debug("Got a redirect status code [" + status + "] new location = [" + locationString + "]");
1434             }
1435 
1436             if (allowedRedirects == 0) {
1437                 throw new FailingHttpStatusCodeException("Too much redirect for "
1438                     + webResponse.getWebRequest().getUrl(), webResponse);
1439             }
1440 
1441             if (status == HttpStatus.SC_MOVED_PERMANENTLY
1442                     || status == HttpStatus.SC_MOVED_TEMPORARILY
1443                     || status == HttpStatus.SC_SEE_OTHER) {
1444                 final WebRequest wrs = new WebRequest(newUrl, HttpMethod.GET);
1445                 for (final Map.Entry<String, String> entry : webRequest.getAdditionalHeaders().entrySet()) {
1446                     wrs.setAdditionalHeader(entry.getKey(), entry.getValue());
1447                 }
1448                 return loadWebResponseFromWebConnection(wrs, allowedRedirects - 1, UTF_8);
1449             }
1450             else if (status == HttpStatus.SC_TEMPORARY_REDIRECT
1451                         || status == 308) {
1452                 final WebRequest wrs = new WebRequest(newUrl, webRequest.getHttpMethod());
1453                 wrs.setRequestParameters(parameters);
1454                 for (final Map.Entry<String, String> entry : webRequest.getAdditionalHeaders().entrySet()) {
1455                     wrs.setAdditionalHeader(entry.getKey(), entry.getValue());
1456                 }
1457                 return loadWebResponseFromWebConnection(wrs, allowedRedirects - 1, UTF_8);
1458             }
1459         }
1460 
1461         return webResponse;
1462     }
1463 
1464     /**
1465      * Adds the headers that are sent with every request to the specified {@link WebRequest} instance.
1466      * @param wrs the <tt>WebRequestSettings</tt> instance to modify
1467      */
1468     private void addDefaultHeaders(final WebRequest wrs) {
1469         // Add standard HtmlUnit headers.
1470         if (!wrs.isAdditionalHeader(HttpHeader.ACCEPT_LANGUAGE)) {
1471             wrs.setAdditionalHeader(HttpHeader.ACCEPT_LANGUAGE, getBrowserVersion().getBrowserLanguage());
1472         }
1473         if (getBrowserVersion().hasFeature(HTTP_HEADER_UPGRADE_INSECURE_REQUEST)
1474                 && !wrs.isAdditionalHeader(HttpHeader.UPGRADE_INSECURE_REQUESTS)) {
1475             wrs.setAdditionalHeader(HttpHeader.UPGRADE_INSECURE_REQUESTS, "1");
1476         }
1477         // Add user-specified headers last so that they can override HtmlUnit defaults.
1478         wrs.getAdditionalHeaders().putAll(requestHeaders_);
1479     }
1480 
1481     /**
1482      * Returns an immutable list of open web windows (whether they are top level windows or not).
1483      * This is a snapshot; future changes are not reflected by this list.
1484      *
1485      * @return an immutable list of open web windows (whether they are top level windows or not)
1486      * @see #getWebWindowByName(String)
1487      * @see #getTopLevelWindows()
1488      */
1489     public List<WebWindow> getWebWindows() {
1490         return Collections.unmodifiableList(new ArrayList<>(windows_));
1491     }
1492 
1493     /**
1494      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
1495      *
1496      * Returns true if the list of WebWindows contains the provided one.
1497      * This method is there to improve the performance of some internal checks because
1498      * calling getWebWindows().contains(.) creates some objects without any need.
1499      *
1500      * @param webWindow the window to check
1501      * @return true or false
1502      */
1503     public boolean containsWebWindow(final WebWindow webWindow) {
1504         return windows_.contains(webWindow);
1505     }
1506 
1507     /**
1508      * Returns an immutable list of open top level windows.
1509      * This is a snapshot; future changes are not reflected by this list.
1510      *
1511      * @return an immutable list of open top level windows
1512      * @see #getWebWindowByName(String)
1513      * @see #getWebWindows()
1514      */
1515     public List<TopLevelWindow> getTopLevelWindows() {
1516         return Collections.unmodifiableList(new ArrayList<>(topLevelWindows_));
1517     }
1518 
1519     /**
1520      * Sets the handler to be used whenever a refresh is triggered. Refer
1521      * to the documentation for {@link RefreshHandler} for more details.
1522      * @param handler the new handler
1523      */
1524     public void setRefreshHandler(final RefreshHandler handler) {
1525         if (handler == null) {
1526             refreshHandler_ = new NiceRefreshHandler(2);
1527         }
1528         else {
1529             refreshHandler_ = handler;
1530         }
1531     }
1532 
1533     /**
1534      * Returns the current refresh handler.
1535      * The default refresh handler is a {@link NiceRefreshHandler NiceRefreshHandler(2)}.
1536      * @return the current RefreshHandler
1537      */
1538     public RefreshHandler getRefreshHandler() {
1539         return refreshHandler_;
1540     }
1541 
1542     /**
1543      * Sets the script pre processor for this webclient.
1544      * @param scriptPreProcessor the new preprocessor or null if none is specified
1545      */
1546     public void setScriptPreProcessor(final ScriptPreProcessor scriptPreProcessor) {
1547         scriptPreProcessor_ = scriptPreProcessor;
1548     }
1549 
1550     /**
1551      * Returns the script pre processor for this webclient.
1552      * @return the pre processor or null of one hasn't been set
1553      */
1554     public ScriptPreProcessor getScriptPreProcessor() {
1555         return scriptPreProcessor_;
1556     }
1557 
1558     /**
1559      * Sets the active X object map for this webclient. The <code>Map</code> is used to map the
1560      * string passed into the <code>ActiveXObject</code> constructor to a java class name. Therefore
1561      * you can emulate <code>ActiveXObject</code>s in a web page's JavaScript by mapping the object
1562      * name to a java class to emulate the active X object.
1563      * @param activeXObjectMap the new preprocessor or null if none is specified
1564      */
1565     public void setActiveXObjectMap(final Map<String, String> activeXObjectMap) {
1566         activeXObjectMap_ = activeXObjectMap;
1567     }
1568 
1569     /**
1570      * Returns the active X object map for this webclient.
1571      * @return the active X object map
1572      */
1573     public Map<String, String> getActiveXObjectMap() {
1574         return activeXObjectMap_;
1575     }
1576 
1577     /**
1578      * Returns the MSXML ActiveX object factory (if supported).
1579      * @return the msxmlActiveXObjectFactory
1580      */
1581     public MSXMLActiveXObjectFactory getMSXMLActiveXObjectFactory() {
1582         return msxmlActiveXObjectFactory_;
1583     }
1584 
1585     /**
1586      * Sets the listener for messages generated by the HTML parser.
1587      * @param listener the new listener, {@code null} if messages should be totally ignored
1588      */
1589     public void setHTMLParserListener(final HTMLParserListener listener) {
1590         htmlParserListener_ = listener;
1591     }
1592 
1593     /**
1594      * Gets the configured listener for messages generated by the HTML parser.
1595      * @return {@code null} if no listener is defined (default value)
1596      */
1597     public HTMLParserListener getHTMLParserListener() {
1598         return htmlParserListener_;
1599     }
1600 
1601     /**
1602      * Returns the CSS error handler used by this web client when CSS problems are encountered.
1603      * @return the CSS error handler used by this web client when CSS problems are encountered
1604      * @see DefaultCssErrorHandler
1605      * @see SilentCssErrorHandler
1606      */
1607     public ErrorHandler getCssErrorHandler() {
1608         return cssErrorHandler_;
1609     }
1610 
1611     /**
1612      * Sets the CSS error handler used by this web client when CSS problems are encountered.
1613      * @param cssErrorHandler the CSS error handler used by this web client when CSS problems are encountered
1614      * @see DefaultCssErrorHandler
1615      * @see SilentCssErrorHandler
1616      */
1617     public void setCssErrorHandler(final ErrorHandler cssErrorHandler) {
1618         WebAssert.notNull("cssErrorHandler", cssErrorHandler);
1619         cssErrorHandler_ = cssErrorHandler;
1620     }
1621 
1622     /**
1623      * Sets the number of milliseconds that a script is allowed to execute before being terminated.
1624      * A value of 0 or less means no timeout.
1625      *
1626      * @param timeout the timeout value, in milliseconds
1627      */
1628     public void setJavaScriptTimeout(final long timeout) {
1629         scriptEngine_.setJavaScriptTimeout(timeout);
1630     }
1631 
1632     /**
1633      * Returns the number of milliseconds that a script is allowed to execute before being terminated.
1634      * A value of 0 or less means no timeout.
1635      *
1636      * @return the timeout value, in milliseconds
1637      */
1638     public long getJavaScriptTimeout() {
1639         return scriptEngine_.getJavaScriptTimeout();
1640     }
1641 
1642     /**
1643      * Gets the current listener for encountered incorrectness (except HTML parsing messages that
1644      * are handled by the HTML parser listener). Default value is an instance of
1645      * {@link IncorrectnessListenerImpl}.
1646      * @return the current listener (not {@code null})
1647      */
1648     public IncorrectnessListener getIncorrectnessListener() {
1649         return incorrectnessListener_;
1650     }
1651 
1652     /**
1653      * Returns the current HTML incorrectness listener.
1654      * @param listener the new value (not {@code null})
1655      */
1656     public void setIncorrectnessListener(final IncorrectnessListener listener) {
1657         if (listener == null) {
1658             throw new NullPointerException("Null incorrectness listener.");
1659         }
1660         incorrectnessListener_ = listener;
1661     }
1662 
1663     /**
1664      * Returns the WebConsole.
1665      * @return the web console
1666      */
1667     public WebConsole getWebConsole() {
1668         if (webConsole_ == null) {
1669             webConsole_ = new WebConsole();
1670         }
1671         return webConsole_;
1672     }
1673 
1674     /**
1675      * Gets the current AJAX controller.
1676      * @return the controller
1677      */
1678     public AjaxController getAjaxController() {
1679         return ajaxController_;
1680     }
1681 
1682     /**
1683      * Sets the current AJAX controller.
1684      * @param newValue the controller
1685      */
1686     public void setAjaxController(final AjaxController newValue) {
1687         if (newValue == null) {
1688             throw new NullPointerException();
1689         }
1690         ajaxController_ = newValue;
1691     }
1692 
1693     /**
1694      * Sets the attachment handler.
1695      * @param handler the new attachment handler
1696      */
1697     public void setAttachmentHandler(final AttachmentHandler handler) {
1698         attachmentHandler_ = handler;
1699     }
1700 
1701     /**
1702      * Returns the current attachment handler.
1703      * @return the current attachment handler
1704      */
1705     public AttachmentHandler getAttachmentHandler() {
1706         return attachmentHandler_;
1707     }
1708 
1709     /**
1710      * Sets the WebStart handler.
1711      * @param handler the new WebStart handler
1712      */
1713     public void setWebStartHandler(final WebStartHandler handler) {
1714         webStartHandler_ = handler;
1715     }
1716 
1717     /**
1718      * Returns the current WebStart handler.
1719      * @return the current WebStart handler
1720      */
1721     public WebStartHandler getWebStartHandler() {
1722         return webStartHandler_;
1723     }
1724 
1725     /**
1726      * Sets the applet confirm handler.
1727      * @param handler the new applet confirm handler handler
1728      */
1729     public void setAppletConfirmHandler(final AppletConfirmHandler handler) {
1730         appletConfirmHandler_ = handler;
1731     }
1732 
1733     /**
1734      * Returns the current applet confirm handler.
1735      * @return the current applet confirm handler
1736      */
1737     public AppletConfirmHandler getAppletConfirmHandler() {
1738         return appletConfirmHandler_;
1739     }
1740 
1741     /**
1742      * Sets the onbeforeunload handler for this webclient.
1743      * @param onbeforeunloadHandler the new onbeforeunloadHandler or null if none is specified
1744      */
1745     public void setOnbeforeunloadHandler(final OnbeforeunloadHandler onbeforeunloadHandler) {
1746         onbeforeunloadHandler_ = onbeforeunloadHandler;
1747     }
1748 
1749     /**
1750      * Returns the onbeforeunload handler for this webclient.
1751      * @return the onbeforeunload handler or null if one hasn't been set
1752      */
1753     public OnbeforeunloadHandler getOnbeforeunloadHandler() {
1754         return onbeforeunloadHandler_;
1755     }
1756 
1757     /**
1758      * Gets the cache currently being used.
1759      * @return the cache (may not be null)
1760      */
1761     public Cache getCache() {
1762         return cache_;
1763     }
1764 
1765     /**
1766      * Sets the cache to use.
1767      * @param cache the new cache (must not be {@code null})
1768      */
1769     public void setCache(final Cache cache) {
1770         if (cache == null) {
1771             throw new IllegalArgumentException("cache should not be null!");
1772         }
1773         cache_ = cache;
1774     }
1775 
1776     /**
1777      * Keeps track of the current window. Inspired by WebTest's logic to track the current response.
1778      */
1779     private static final class CurrentWindowTracker implements WebWindowListener, Serializable {
1780         private final WebClient webClient_;
1781 
1782         private CurrentWindowTracker(final WebClient webClient) {
1783             webClient_ = webClient;
1784         }
1785 
1786         /**
1787          * {@inheritDoc}
1788          */
1789         @Override
1790         public void webWindowClosed(final WebWindowEvent event) {
1791             final WebWindow window = event.getWebWindow();
1792             if (window instanceof TopLevelWindow) {
1793                 webClient_.topLevelWindows_.remove(window);
1794                 if (window == webClient_.getCurrentWindow()) {
1795                     if (webClient_.topLevelWindows_.isEmpty()) {
1796                         // Must always have at least window, and there are no top-level windows left; must create one.
1797                         final TopLevelWindow newWindow = new TopLevelWindow("", webClient_);
1798                         webClient_.topLevelWindows_.push(newWindow);
1799                         webClient_.setCurrentWindow(newWindow);
1800                     }
1801                     else {
1802                         // The current window is now the previous top-level window.
1803                         webClient_.setCurrentWindow(webClient_.topLevelWindows_.peek());
1804                     }
1805                 }
1806             }
1807             else if (window == webClient_.getCurrentWindow()) {
1808                 // The current window is now the last top-level window.
1809                 webClient_.setCurrentWindow(webClient_.topLevelWindows_.peek());
1810             }
1811         }
1812 
1813         /**
1814          * {@inheritDoc}
1815          */
1816         @Override
1817         public void webWindowContentChanged(final WebWindowEvent event) {
1818             final WebWindow window = event.getWebWindow();
1819             boolean use = false;
1820             if (window instanceof DialogWindow) {
1821                 use = true;
1822             }
1823             else if (window instanceof TopLevelWindow) {
1824                 use = event.getOldPage() == null;
1825             }
1826             else if (window instanceof FrameWindow) {
1827                 final FrameWindow fw = (FrameWindow) window;
1828                 final String enclosingPageState = fw.getEnclosingPage().getDocumentElement().getReadyState();
1829                 final URL frameUrl = fw.getEnclosedPage().getUrl();
1830                 if (!DomNode.READY_STATE_COMPLETE.equals(enclosingPageState) || frameUrl == URL_ABOUT_BLANK) {
1831                     return;
1832                 }
1833 
1834                 // now looks at the visibility of the frame window
1835                 final BaseFrameElement frameElement = fw.getFrameElement();
1836                 if (frameElement.isDisplayed()) {
1837                     final Object element = frameElement.getScriptableObject();
1838                     final HTMLElement htmlElement = (HTMLElement) element;
1839                     final ComputedCSSStyleDeclaration style =
1840                             htmlElement.getWindow().getComputedStyle(htmlElement, null);
1841                     use = style.getCalculatedWidth(false, false) != 0
1842                             && style.getCalculatedHeight(false, false) != 0;
1843                 }
1844             }
1845             if (use) {
1846                 webClient_.setCurrentWindow(window);
1847             }
1848         }
1849 
1850         /**
1851          * {@inheritDoc}
1852          */
1853         @Override
1854         public void webWindowOpened(final WebWindowEvent event) {
1855             final WebWindow window = event.getWebWindow();
1856             if (window instanceof TopLevelWindow) {
1857                 final TopLevelWindow tlw = (TopLevelWindow) window;
1858                 webClient_.topLevelWindows_.push(tlw);
1859             }
1860             // Page is not loaded yet, don't set it now as current window.
1861         }
1862     }
1863 
1864     /**
1865      * Closes all opened windows, stopping all background JavaScript processing.
1866      *
1867      * {@inheritDoc}
1868      */
1869     @Override
1870     public void close() {
1871         // NB: this implementation is too simple as a new TopLevelWindow may be opened by
1872         // some JS script while we are closing the others
1873         final List<TopLevelWindow> topWindows = new ArrayList<>(topLevelWindows_);
1874         for (final TopLevelWindow topWindow : topWindows) {
1875             if (topLevelWindows_.contains(topWindow)) {
1876                 try {
1877                     topWindow.close();
1878                 }
1879                 catch (final Exception e) {
1880                     LOG.error("Exception while closing a topLevelWindow", e);
1881                 }
1882             }
1883         }
1884 
1885         // do this after closing the windows, otherwise some unload event might
1886         // start a new window that will start the thread again
1887         if (scriptEngine_ != null) {
1888             try {
1889                 scriptEngine_.shutdown();
1890             }
1891             catch (final Exception e) {
1892                 LOG.error("Exception while shutdown the scriptEngine", e);
1893             }
1894         }
1895 
1896         try {
1897             webConnection_.close();
1898         }
1899         catch (final Exception e) {
1900             LOG.error("Exception while closing the connection", e);
1901         }
1902 
1903         cache_.clear();
1904     }
1905 
1906     /**
1907      * <p><span style="color:red">Experimental API: May be changed in next release
1908      * and may not yet work perfectly!</span></p>
1909      *
1910      * <p>This method blocks until all background JavaScript tasks have finished executing. Background
1911      * JavaScript tasks are JavaScript tasks scheduled for execution via <tt>window.setTimeout</tt>,
1912      * <tt>window.setInterval</tt> or asynchronous <tt>XMLHttpRequest</tt>.</p>
1913      *
1914      * <p>If a job is scheduled to begin executing after <tt>(now + timeoutMillis)</tt>, this method will
1915      * wait for <tt>timeoutMillis</tt> milliseconds and then return a value greater than <tt>0</tt>. This
1916      * method will never block longer than <tt>timeoutMillis</tt> milliseconds.</p>
1917      *
1918      * <p>Use this method instead of {@link #waitForBackgroundJavaScriptStartingBefore(long)} if you
1919      * don't know when your background JavaScript is supposed to start executing, but you're fairly sure
1920      * that you know how long it should take to finish executing.</p>
1921      *
1922      * @param timeoutMillis the maximum amount of time to wait (in milliseconds)
1923      * @return the number of background JavaScript jobs still executing or waiting to be executed when this
1924      *         method returns; will be <tt>0</tt> if there are no jobs left to execute
1925      */
1926     public int waitForBackgroundJavaScript(final long timeoutMillis) {
1927         int count = 0;
1928         final long endTime = System.currentTimeMillis() + timeoutMillis;
1929         for (Iterator<WeakReference<JavaScriptJobManager>> i = jobManagers_.iterator(); i.hasNext();) {
1930             final JavaScriptJobManager jobManager;
1931             final WeakReference<JavaScriptJobManager> reference;
1932             try {
1933                 reference = i.next();
1934                 jobManager = reference.get();
1935                 if (jobManager == null) {
1936                     i.remove();
1937                     continue;
1938                 }
1939             }
1940             catch (final ConcurrentModificationException e) {
1941                 i = jobManagers_.iterator();
1942                 count = 0;
1943                 continue;
1944             }
1945 
1946             final long newTimeout = endTime - System.currentTimeMillis();
1947             count += jobManager.waitForJobs(newTimeout);
1948         }
1949         if (count != getAggregateJobCount()) {
1950             final long newTimeout = endTime - System.currentTimeMillis();
1951             return waitForBackgroundJavaScript(newTimeout);
1952         }
1953         return count;
1954     }
1955 
1956     /**
1957      * <p><span style="color:red">Experimental API: May be changed in next release
1958      * and may not yet work perfectly!</span></p>
1959      *
1960      * <p>This method blocks until all background JavaScript tasks scheduled to start executing before
1961      * <tt>(now + delayMillis)</tt> have finished executing. Background JavaScript tasks are JavaScript
1962      * tasks scheduled for execution via <tt>window.setTimeout</tt>, <tt>window.setInterval</tt> or
1963      * asynchronous <tt>XMLHttpRequest</tt>.</p>
1964      *
1965      * <p>If there is no background JavaScript task currently executing, and there is no background JavaScript
1966      * task scheduled to start executing within the specified time, this method returns immediately -- even
1967      * if there are tasks scheduled to be executed after <tt>(now + delayMillis)</tt>.</p>
1968      *
1969      * <p>Note that the total time spent executing a background JavaScript task is never known ahead of
1970      * time, so this method makes no guarantees as to how long it will block.</p>
1971      *
1972      * <p>Use this method instead of {@link #waitForBackgroundJavaScript(long)} if you know roughly when
1973      * your background JavaScript is supposed to start executing, but you're not necessarily sure how long
1974      * it will take to execute.</p>
1975      *
1976      * @param delayMillis the delay which determines the background tasks to wait for (in milliseconds)
1977      * @return the number of background JavaScript jobs still executing or waiting to be executed when this
1978      *         method returns; will be <tt>0</tt> if there are no jobs left to execute
1979      */
1980     public int waitForBackgroundJavaScriptStartingBefore(final long delayMillis) {
1981         int count = 0;
1982         final long endTime = System.currentTimeMillis() + delayMillis;
1983         for (Iterator<WeakReference<JavaScriptJobManager>> i = jobManagers_.iterator(); i.hasNext();) {
1984             final JavaScriptJobManager jobManager;
1985             final WeakReference<JavaScriptJobManager> reference;
1986             try {
1987                 reference = i.next();
1988                 jobManager = reference.get();
1989                 if (jobManager == null) {
1990                     i.remove();
1991                     continue;
1992                 }
1993             }
1994             catch (final ConcurrentModificationException e) {
1995                 i = jobManagers_.iterator();
1996                 count = 0;
1997                 continue;
1998             }
1999             final long newDelay = endTime - System.currentTimeMillis();
2000             count += jobManager.waitForJobsStartingBefore(newDelay);
2001         }
2002         if (count != getAggregateJobCount()) {
2003             final long newDelay = endTime - System.currentTimeMillis();
2004             return waitForBackgroundJavaScriptStartingBefore(newDelay);
2005         }
2006         return count;
2007     }
2008 
2009     /**
2010      * Returns the aggregate background JavaScript job count across all windows.
2011      * @return the aggregate background JavaScript job count across all windows
2012      */
2013     private int getAggregateJobCount() {
2014         int count = 0;
2015         for (Iterator<WeakReference<JavaScriptJobManager>> i = jobManagers_.iterator(); i.hasNext();) {
2016             final JavaScriptJobManager jobManager;
2017             final WeakReference<JavaScriptJobManager> reference;
2018             try {
2019                 reference = i.next();
2020                 jobManager = reference.get();
2021                 if (jobManager == null) {
2022                     i.remove();
2023                     continue;
2024                 }
2025             }
2026             catch (final ConcurrentModificationException e) {
2027                 i = jobManagers_.iterator();
2028                 count = 0;
2029                 continue;
2030             }
2031             final int jobCount = jobManager.getJobCount();
2032             count += jobCount;
2033         }
2034         return count;
2035     }
2036 
2037     /**
2038      * When we deserialize, re-initializie transient fields.
2039      * @param in the object input stream
2040      * @throws IOException if an error occurs
2041      * @throws ClassNotFoundException if an error occurs
2042      */
2043     private void readObject(final ObjectInputStream in) throws IOException, ClassNotFoundException {
2044         in.defaultReadObject();
2045 
2046         webConnection_ = createWebConnection();
2047         scriptEngine_ = new JavaScriptEngine(this);
2048         jobManagers_ = Collections.synchronizedList(new ArrayList<WeakReference<JavaScriptJobManager>>());
2049 
2050         if (getBrowserVersion().hasFeature(JS_XML_SUPPORT_VIA_ACTIVEXOBJECT)) {
2051             initMSXMLActiveX();
2052         }
2053     }
2054 
2055     private WebConnection createWebConnection() {
2056         if (GAEUtils.isGaeMode()) {
2057             return new UrlFetchWebConnection(this);
2058         }
2059 
2060         return new HttpWebConnection(this);
2061     }
2062 
2063     private static class LoadJob {
2064         private final WebWindow requestingWindow_;
2065         private final String target_;
2066         private final WebResponse response_;
2067         private final URL urlWithOnlyHashChange_;
2068         private final WeakReference<Page> originalPage_;
2069         private final WebRequest request_;
2070 
2071         LoadJob(final WebRequest request, final WebWindow requestingWindow, final String target,
2072                 final WebResponse response) {
2073             request_ = request;
2074             requestingWindow_ = requestingWindow;
2075             target_ = target;
2076             response_ = response;
2077             urlWithOnlyHashChange_ = null;
2078             originalPage_ = new WeakReference<>(requestingWindow.getEnclosedPage());
2079         }
2080 
2081         LoadJob(final WebRequest request, final WebWindow requestingWindow, final String target,
2082                 final URL urlWithOnlyHashChange) {
2083             request_ = request;
2084             requestingWindow_ = requestingWindow;
2085             target_ = target;
2086             response_ = null;
2087             urlWithOnlyHashChange_ = urlWithOnlyHashChange;
2088             originalPage_ = new WeakReference<>(requestingWindow.getEnclosedPage());
2089         }
2090 
2091         public boolean isOutdated() {
2092             if (target_ != null && !target_.isEmpty()) {
2093                 return false;
2094             }
2095             else if (requestingWindow_.isClosed()) {
2096                 return true;
2097             }
2098             else if (requestingWindow_.getEnclosedPage() != originalPage_.get()) {
2099                 return true;
2100             }
2101 
2102             return false;
2103         }
2104     }
2105 
2106     private final List<LoadJob> loadQueue_ = new ArrayList<>();
2107 
2108     /**
2109      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2110      *
2111      * Perform the downloads and stores it for loading later into a window.
2112      * In the future downloads should be performed in parallel in separated threads.
2113      * TODO: refactor it before next release.
2114      * @param requestingWindow the window from which the request comes
2115      * @param target the name of the target window
2116      * @param request the request to perform
2117      * @param checkHash if true check for hashChenage
2118      * @param forceLoad if true always load the request even if there is already the same in the queue
2119      * @param description information about the origin of the request. Useful for debugging.
2120      */
2121     public void download(final WebWindow requestingWindow, final String target,
2122         final WebRequest request, final boolean checkHash, final boolean forceLoad, final String description) {
2123         final WebWindow win = resolveWindow(requestingWindow, target);
2124         final URL url = request.getUrl();
2125         boolean justHashJump = false;
2126 
2127         if (win != null && HttpMethod.POST != request.getHttpMethod()) {
2128             final Page page = win.getEnclosedPage();
2129             if (page != null) {
2130                 if (page.isHtmlPage() && !((HtmlPage) page).isOnbeforeunloadAccepted()) {
2131                     return;
2132                 }
2133 
2134                 if (checkHash) {
2135                     final URL current = page.getUrl();
2136                     justHashJump =
2137                             HttpMethod.GET == request.getHttpMethod()
2138                             && UrlUtils.sameFile(url, current)
2139                             && null != url.getRef();
2140                 }
2141             }
2142         }
2143 
2144         synchronized (loadQueue_) {
2145             // verify if this load job doesn't already exist
2146             for (final LoadJob loadJob : loadQueue_) {
2147                 if (loadJob.response_ == null) {
2148                     continue;
2149                 }
2150                 final WebRequest otherRequest = loadJob.request_;
2151                 final URL otherUrl = otherRequest.getUrl();
2152 
2153                 // TODO: investigate but it seems that IE considers query string too but not FF
2154                 if (!forceLoad
2155                     && url.getPath().equals(otherUrl.getPath()) // fail fast
2156                     && url.toString().equals(otherUrl.toString())
2157                     && request.getRequestParameters().equals(otherRequest.getRequestParameters())
2158                     && StringUtils.equals(request.getRequestBody(), otherRequest.getRequestBody())) {
2159                     return; // skip it;
2160                 }
2161             }
2162         }
2163 
2164         final LoadJob loadJob;
2165         if (justHashJump) {
2166             loadJob = new LoadJob(request, requestingWindow, target, url);
2167         }
2168         else {
2169             try {
2170                 final WebResponse response = loadWebResponse(request);
2171                 loadJob = new LoadJob(request, requestingWindow, target, response);
2172             }
2173             catch (final IOException e) {
2174                 throw new RuntimeException(e);
2175             }
2176         }
2177         synchronized (loadQueue_) {
2178             loadQueue_.add(loadJob);
2179         }
2180     }
2181 
2182     /**
2183      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2184      *
2185      * Loads downloaded responses into the corresponding windows.
2186      * TODO: refactor it before next release.
2187      * @throws IOException in case of exception
2188      * @throws FailingHttpStatusCodeException in case of exception
2189      */
2190     public void loadDownloadedResponses() throws FailingHttpStatusCodeException, IOException {
2191         final List<LoadJob> queue;
2192 
2193         // synchronize access to the loadQueue_,
2194         // to be sure no job is ignored
2195         synchronized (loadQueue_) {
2196             if (loadQueue_.isEmpty()) {
2197                 return;
2198             }
2199             queue = new ArrayList<>(loadQueue_);
2200             loadQueue_.clear();
2201         }
2202 
2203         final HashSet<WebWindow> updatedWindows = new HashSet<>();
2204         for (int i = queue.size() - 1; i >= 0; --i) {
2205             final LoadJob loadJob = queue.get(i);
2206             if (loadJob.isOutdated()) {
2207                 LOG.info("No usage of download: " + loadJob);
2208                 continue;
2209             }
2210 
2211             final WebWindow window = resolveWindow(loadJob.requestingWindow_, loadJob.target_);
2212             if (!updatedWindows.contains(window)) {
2213                 final WebWindow win = openTargetWindow(loadJob.requestingWindow_, loadJob.target_, "_self");
2214                 if (loadJob.urlWithOnlyHashChange_ != null) {
2215                     final HtmlPage page = (HtmlPage) loadJob.requestingWindow_.getEnclosedPage();
2216                     final String oldURL = page.getUrl().toExternalForm();
2217 
2218                     // update request url
2219                     final WebRequest req = page.getWebResponse().getWebRequest();
2220                     req.setUrl(loadJob.urlWithOnlyHashChange_);
2221 
2222                     // update location.hash
2223                     final Window jsWindow = (Window) win.getScriptableObject();
2224                     if (null != jsWindow) {
2225                         final Location location = jsWindow.getLocation();
2226                         location.setHash(oldURL, loadJob.urlWithOnlyHashChange_.getRef());
2227                     }
2228 
2229                     // add to history
2230                     win.getHistory().addPage(page);
2231                 }
2232                 else {
2233                     final Page pageBeforeLoad = win.getEnclosedPage();
2234                     loadWebResponseInto(loadJob.response_, win);
2235 
2236                     // start execution here.
2237                     if (scriptEngine_ != null) {
2238                         scriptEngine_.registerWindowAndMaybeStartEventLoop(win);
2239                     }
2240 
2241                     if (pageBeforeLoad != win.getEnclosedPage()) {
2242                         updatedWindows.add(win);
2243                     }
2244 
2245                     // check and report problems if needed
2246                     throwFailingHttpStatusCodeExceptionIfNecessary(loadJob.response_);
2247                 }
2248             }
2249             else {
2250                 LOG.info("No usage of download: " + loadJob);
2251             }
2252         }
2253     }
2254 
2255     /**
2256      * Returns the options object of this WebClient.
2257      * @return the options object
2258      */
2259     public WebClientOptions getOptions() {
2260         return options_;
2261     }
2262 
2263     /**
2264      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2265      *
2266      * Returns the internals object of this WebClient.
2267      * @return the internals object
2268      */
2269     public WebClientInternals getInternals() {
2270         return internals_;
2271     }
2272 
2273     /**
2274      * Gets the holder for the different storages.
2275      * <p><span style="color:red">Experimental API: May be changed in next release!</span></p>
2276      * @return the holder
2277      */
2278     public StorageHolder getStorageHolder() {
2279         return storageHolder_;
2280     }
2281 
2282     /**
2283      * Returns the currently configured cookies applicable to the specified URL, in an unmodifiable set.
2284      * If disabled, this returns an empty set.
2285      * @param url the URL on which to filter the returned cookies
2286      * @return the currently configured cookies applicable to the specified URL, in an unmodifiable set
2287      */
2288     public synchronized Set<Cookie> getCookies(final URL url) {
2289         final CookieManager cookieManager = getCookieManager();
2290 
2291         if (!cookieManager.isCookiesEnabled()) {
2292             return Collections.<Cookie>emptySet();
2293         }
2294 
2295         final URL normalizedUrl = cookieManager.replaceForCookieIfNecessary(url);
2296 
2297         final String host = normalizedUrl.getHost();
2298         // URLs like "about:blank" don't have cookies and we need to catch these
2299         // cases here before HttpClient complains
2300         if (host.isEmpty()) {
2301             return Collections.emptySet();
2302         }
2303 
2304         final String path = normalizedUrl.getPath();
2305         final String protocol = normalizedUrl.getProtocol();
2306         final boolean secure = "https".equals(protocol);
2307 
2308         final int port = cookieManager.getPort(normalizedUrl);
2309 
2310         // discard expired cookies
2311         cookieManager.clearExpired(new Date());
2312 
2313         final List<org.apache.http.cookie.Cookie> all = Cookie.toHttpClient(cookieManager.getCookies());
2314         final List<org.apache.http.cookie.Cookie> matches = new ArrayList<>();
2315 
2316         if (all.size() > 0) {
2317             final CookieOrigin cookieOrigin = new CookieOrigin(host, port, path, secure);
2318             final CookieSpec cookieSpec = new HtmlUnitBrowserCompatCookieSpec(getBrowserVersion());
2319             for (final org.apache.http.cookie.Cookie cookie : all) {
2320                 if (cookieSpec.match(cookie, cookieOrigin)) {
2321                     matches.add(cookie);
2322                 }
2323             }
2324         }
2325 
2326         final Set<Cookie> cookies = new LinkedHashSet<>();
2327         cookies.addAll(Cookie.fromHttpClient(matches));
2328         return Collections.unmodifiableSet(cookies);
2329     }
2330 
2331     /**
2332      * Parses the given cookie and adds this to our cookie store.
2333      * @param cookieString the string to parse
2334      * @param pageUrl the url of the page that likes to set the cookie
2335      * @param origin the requester
2336      */
2337     public void addCookie(final String cookieString, final URL pageUrl, final Object origin) {
2338         final BrowserVersion browserVersion = getBrowserVersion();
2339         final CookieManager cookieManager = getCookieManager();
2340         if (cookieManager.isCookiesEnabled()) {
2341             final CharArrayBuffer buffer = new CharArrayBuffer(cookieString.length() + 22);
2342             buffer.append("Set-Cookie: ");
2343             buffer.append(cookieString);
2344 
2345             final CookieSpec cookieSpec = new HtmlUnitBrowserCompatCookieSpec(browserVersion);
2346 
2347             try {
2348                 final List<org.apache.http.cookie.Cookie> cookies =
2349                         cookieSpec.parse(new BufferedHeader(buffer), cookieManager.buildCookieOrigin(pageUrl));
2350 
2351                 for (org.apache.http.cookie.Cookie cookie : cookies) {
2352                     final Cookie htmlUnitCookie = new Cookie((ClientCookie) cookie);
2353                     cookieManager.addCookie(htmlUnitCookie);
2354 
2355                     if (LOG.isDebugEnabled()) {
2356                         LOG.debug("Added cookie: '" + cookieString + "'");
2357                     }
2358                 }
2359             }
2360             catch (final MalformedCookieException e) {
2361                 getIncorrectnessListener().notify("set-cookie http-equiv meta tag: invalid cookie '"
2362                         + cookieString + "'; reason: '" + e.getMessage() + "'.", origin);
2363             }
2364         }
2365         else if (LOG.isDebugEnabled()) {
2366             LOG.debug("Skipped adding cookie: '" + cookieString + "'");
2367         }
2368     }
2369 }