View Javadoc
1   /*
2    * Copyright (c) 2002-2017 Gargoyle Software Inc.
3    *
4    * Licensed under the Apache License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    * http://www.apache.org/licenses/LICENSE-2.0
8    *
9    * Unless required by applicable law or agreed to in writing, software
10   * distributed under the License is distributed on an "AS IS" BASIS,
11   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12   * See the License for the specific language governing permissions and
13   * limitations under the License.
14   */
15  package com.gargoylesoftware.htmlunit.html;
16  
17  import static com.gargoylesoftware.htmlunit.BrowserVersionFeatures.EVENT_FOCUS_FOCUS_IN_BLUR_OUT;
18  import static com.gargoylesoftware.htmlunit.BrowserVersionFeatures.EVENT_FOCUS_IN_FOCUS_OUT_BLUR;
19  import static com.gargoylesoftware.htmlunit.BrowserVersionFeatures.FOCUS_BODY_ELEMENT_AT_START;
20  import static com.gargoylesoftware.htmlunit.BrowserVersionFeatures.JS_CALL_RESULT_IS_LAST_RETURN_VALUE;
21  import static com.gargoylesoftware.htmlunit.BrowserVersionFeatures.JS_DEFERRED;
22  import static com.gargoylesoftware.htmlunit.BrowserVersionFeatures.PAGE_SELECTION_RANGE_FROM_SELECTABLE_TEXT_INPUT;
23  import static com.gargoylesoftware.htmlunit.BrowserVersionFeatures.URL_MISSING_SLASHES;
24  import static java.nio.charset.StandardCharsets.ISO_8859_1;
25  
26  import java.io.File;
27  import java.io.IOException;
28  import java.io.ObjectInputStream;
29  import java.io.ObjectOutputStream;
30  import java.io.Serializable;
31  import java.net.MalformedURLException;
32  import java.net.URL;
33  import java.nio.charset.Charset;
34  import java.util.ArrayList;
35  import java.util.Arrays;
36  import java.util.Collection;
37  import java.util.Collections;
38  import java.util.Comparator;
39  import java.util.HashMap;
40  import java.util.LinkedHashSet;
41  import java.util.List;
42  import java.util.Locale;
43  import java.util.Map;
44  import java.util.SortedSet;
45  import java.util.TreeSet;
46  
47  import org.apache.commons.lang3.StringUtils;
48  import org.apache.commons.logging.Log;
49  import org.apache.commons.logging.LogFactory;
50  import org.apache.http.HttpStatus;
51  import org.w3c.dom.Attr;
52  import org.w3c.dom.Comment;
53  import org.w3c.dom.DOMConfiguration;
54  import org.w3c.dom.DOMException;
55  import org.w3c.dom.DOMImplementation;
56  import org.w3c.dom.Document;
57  import org.w3c.dom.DocumentType;
58  import org.w3c.dom.Element;
59  import org.w3c.dom.EntityReference;
60  import org.w3c.dom.ProcessingInstruction;
61  import org.w3c.dom.ranges.Range;
62  
63  import com.gargoylesoftware.htmlunit.Cache;
64  import com.gargoylesoftware.htmlunit.ElementNotFoundException;
65  import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException;
66  import com.gargoylesoftware.htmlunit.History;
67  import com.gargoylesoftware.htmlunit.OnbeforeunloadHandler;
68  import com.gargoylesoftware.htmlunit.Page;
69  import com.gargoylesoftware.htmlunit.ScriptResult;
70  import com.gargoylesoftware.htmlunit.SgmlPage;
71  import com.gargoylesoftware.htmlunit.TopLevelWindow;
72  import com.gargoylesoftware.htmlunit.WebAssert;
73  import com.gargoylesoftware.htmlunit.WebClient;
74  import com.gargoylesoftware.htmlunit.WebRequest;
75  import com.gargoylesoftware.htmlunit.WebResponse;
76  import com.gargoylesoftware.htmlunit.WebWindow;
77  import com.gargoylesoftware.htmlunit.html.HTMLParser.HtmlUnitDOMBuilder;
78  import com.gargoylesoftware.htmlunit.html.impl.SelectableTextInput;
79  import com.gargoylesoftware.htmlunit.html.impl.SimpleRange;
80  import com.gargoylesoftware.htmlunit.javascript.AbstractJavaScriptEngine;
81  import com.gargoylesoftware.htmlunit.javascript.JavaScriptEngine;
82  import com.gargoylesoftware.htmlunit.javascript.PostponedAction;
83  import com.gargoylesoftware.htmlunit.javascript.SimpleScriptable;
84  import com.gargoylesoftware.htmlunit.javascript.host.Window;
85  import com.gargoylesoftware.htmlunit.javascript.host.dom.Node;
86  import com.gargoylesoftware.htmlunit.javascript.host.event.BeforeUnloadEvent;
87  import com.gargoylesoftware.htmlunit.javascript.host.event.Event;
88  import com.gargoylesoftware.htmlunit.javascript.host.html.HTMLDocument;
89  import com.gargoylesoftware.htmlunit.protocol.javascript.JavaScriptURLConnection;
90  import com.gargoylesoftware.htmlunit.util.EncodingSniffer;
91  import com.gargoylesoftware.htmlunit.util.UrlUtils;
92  
93  import net.sourceforge.htmlunit.corejs.javascript.Context;
94  import net.sourceforge.htmlunit.corejs.javascript.Function;
95  import net.sourceforge.htmlunit.corejs.javascript.Script;
96  import net.sourceforge.htmlunit.corejs.javascript.Scriptable;
97  import net.sourceforge.htmlunit.corejs.javascript.ScriptableObject;
98  import net.sourceforge.htmlunit.corejs.javascript.Undefined;
99  
100 /**
101  * A representation of an HTML page returned from a server.
102  * <p>
103  * This class provides different methods to access the page's content like
104  * {@link #getForms()}, {@link #getAnchors()}, {@link #getElementById(String)}, ... as well as the
105  * very powerful inherited methods {@link #getByXPath(String)} and {@link #getFirstByXPath(String)}
106  * for fine grained user specific access to child nodes.
107  * </p>
108  * <p>
109  * Child elements allowing user interaction provide methods for this purpose like {@link HtmlAnchor#click()},
110  * {@link HtmlInput#type(String)}, {@link HtmlOption#setSelected(boolean)}, ...
111  * </p>
112  * <p>
113  * HtmlPage instances should not be instantiated directly. They will be returned by {@link WebClient#getPage(String)}
114  * when the content type of the server's response is <code>text/html</code> (or one of its variations).<br>
115  * <br>
116  * <b>Example:</b><br>
117  * <br>
118  * <code>
119  * final HtmlPage page = webClient.{@link WebClient#getPage(String) getPage}("http://mywebsite/some/page.html");
120  * </code>
121  * </p>
122  *
123  * @author <a href="mailto:mbowler@GargoyleSoftware.com">Mike Bowler</a>
124  * @author Alex Nikiforoff
125  * @author Noboru Sinohara
126  * @author David K. Taylor
127  * @author Andreas Hangler
128  * @author <a href="mailto:cse@dynabean.de">Christian Sell</a>
129  * @author Chris Erskine
130  * @author Marc Guillemot
131  * @author Ahmed Ashour
132  * @author Daniel Gredler
133  * @author Dmitri Zoubkov
134  * @author Sudhan Moghe
135  * @author Ethan Glasser-Camp
136  * @author <a href="mailto:tom.anderson@univ.oxon.org">Tom Anderson</a>
137  * @author Ronald Brill
138  * @author Frank Danek
139  * @author Joerg Werner
140  */
141 public class HtmlPage extends SgmlPage {
142 
143     private static final Log LOG = LogFactory.getLog(HtmlPage.class);
144 
145     private static final Comparator<DomElement> documentPositionComparator = new DocumentPositionComparator();
146 
147     private HtmlUnitDOMBuilder builder_;
148     private transient Charset originalCharset_;
149 
150     private Map<String, SortedSet<DomElement>> idMap_
151             = Collections.synchronizedMap(new HashMap<String, SortedSet<DomElement>>());
152     private Map<String, SortedSet<DomElement>> nameMap_
153             = Collections.synchronizedMap(new HashMap<String, SortedSet<DomElement>>());
154 
155     private SortedSet<BaseFrameElement> frameElements_ = new TreeSet<>(documentPositionComparator);
156     private int parserCount_;
157     private int snippetParserCount_;
158     private int inlineSnippetParserCount_;
159     private Collection<HtmlAttributeChangeListener> attributeListeners_;
160     private final Object lock_ = new String(); // used for synchronization
161     private List<PostponedAction> afterLoadActions_ = Collections.synchronizedList(new ArrayList<PostponedAction>());
162     private boolean cleaning_;
163     private HtmlBase base_;
164     private URL baseUrl_;
165     private List<AutoCloseable> autoCloseableList_;
166     private ElementFromPointHandler elementFromPointHandler_;
167     private DomElement elementWithFocus_;
168     private List<Range> selectionRanges_ = new ArrayList<>(3);
169 
170     private static final List<String> TABBABLE_TAGS = Arrays.asList(HtmlAnchor.TAG_NAME, HtmlArea.TAG_NAME,
171             HtmlButton.TAG_NAME, HtmlInput.TAG_NAME, HtmlObject.TAG_NAME, HtmlSelect.TAG_NAME, HtmlTextArea.TAG_NAME);
172     private static final List<String> ACCEPTABLE_TAG_NAMES = Arrays.asList(HtmlAnchor.TAG_NAME, HtmlArea.TAG_NAME,
173             HtmlButton.TAG_NAME, HtmlInput.TAG_NAME, HtmlLabel.TAG_NAME, HtmlLegend.TAG_NAME, HtmlTextArea.TAG_NAME);
174 
175     static class DocumentPositionComparator implements Comparator<DomElement>, Serializable {
176         @Override
177         public int compare(final DomElement elt1, final DomElement elt2) {
178             final short relation = elt1.compareDocumentPosition(elt2);
179             if (relation == 0) {
180                 return 0; // same node
181             }
182             if ((relation & DOCUMENT_POSITION_CONTAINS) != 0 || (relation & DOCUMENT_POSITION_PRECEDING) != 0) {
183                 return 1;
184             }
185 
186             return -1;
187         }
188     }
189 
190     /**
191      * Creates an instance of HtmlPage.
192      * An HtmlPage instance is normally retrieved with {@link WebClient#getPage(String)}.
193      *
194      * @param webResponse the web response that was used to create this page
195      * @param webWindow the window that this page is being loaded into
196      */
197     public HtmlPage(final WebResponse webResponse, final WebWindow webWindow) {
198         super(webResponse, webWindow);
199     }
200 
201     /**
202      * {@inheritDoc}
203      */
204     @Override
205     public HtmlPage getPage() {
206         return this;
207     }
208 
209     /**
210      * {@inheritDoc}
211      */
212     @Override
213     public boolean hasCaseSensitiveTagNames() {
214         return false;
215     }
216 
217     /**
218      * Initialize this page.
219      * @throws IOException if an IO problem occurs
220      * @throws FailingHttpStatusCodeException if the server returns a failing status code AND the property
221      * {@link com.gargoylesoftware.htmlunit.WebClientOptions#setThrowExceptionOnFailingStatusCode(boolean)} is set
222      * to true.
223      */
224     @Override
225     public void initialize() throws IOException, FailingHttpStatusCodeException {
226         final WebWindow enclosingWindow = getEnclosingWindow();
227         final boolean isAboutBlank = getUrl() == WebClient.URL_ABOUT_BLANK;
228         if (isAboutBlank) {
229             // a frame contains first a faked "about:blank" before its real content specified by src gets loaded
230             if (enclosingWindow instanceof FrameWindow
231                     && !((FrameWindow) enclosingWindow).getFrameElement().isContentLoaded()) {
232                 return;
233             }
234 
235             // save the URL that should be used to resolve relative URLs in this page
236             if (enclosingWindow instanceof TopLevelWindow) {
237                 final TopLevelWindow topWindow = (TopLevelWindow) enclosingWindow;
238                 final WebWindow openerWindow = topWindow.getOpener();
239                 if (openerWindow != null && openerWindow.getEnclosedPage() != null) {
240                     baseUrl_ = openerWindow.getEnclosedPage().getWebResponse().getWebRequest().getUrl();
241                 }
242             }
243         }
244         loadFrames();
245 
246         // don't set the ready state if we really load the blank page into the window
247         // see Node.initInlineFrameIfNeeded()
248         if (!isAboutBlank) {
249             if (hasFeature(FOCUS_BODY_ELEMENT_AT_START)) {
250                 setElementWithFocus(getBody());
251             }
252             setReadyState(READY_STATE_COMPLETE);
253             getDocumentElement().setReadyState(READY_STATE_COMPLETE);
254         }
255 
256         executeEventHandlersIfNeeded(Event.TYPE_DOM_DOCUMENT_LOADED);
257         executeDeferredScriptsIfNeeded();
258         setReadyStateOnDeferredScriptsIfNeeded();
259 
260         // frame initialization has a different order
261         boolean isFrameWindow = enclosingWindow instanceof FrameWindow;
262         boolean isFirstPageInFrameWindow = false;
263         if (isFrameWindow) {
264             isFrameWindow = ((FrameWindow) enclosingWindow).getFrameElement() instanceof HtmlFrame;
265 
266             final History hist = enclosingWindow.getHistory();
267             if (hist.getLength() > 0 && WebClient.URL_ABOUT_BLANK == hist.getUrl(0)) {
268                 isFirstPageInFrameWindow = hist.getLength() <= 2;
269             }
270             else {
271                 isFirstPageInFrameWindow = enclosingWindow.getHistory().getLength() < 2;
272             }
273         }
274 
275         if (isFrameWindow && !isFirstPageInFrameWindow) {
276             executeEventHandlersIfNeeded(Event.TYPE_LOAD);
277         }
278 
279         for (final FrameWindow frameWindow : getFrames()) {
280             if (frameWindow.getFrameElement() instanceof HtmlFrame) {
281                 final Page page = frameWindow.getEnclosedPage();
282                 if (page != null && page.isHtmlPage()) {
283                     ((HtmlPage) page).executeEventHandlersIfNeeded(Event.TYPE_LOAD);
284                 }
285             }
286         }
287 
288         if (!isFrameWindow) {
289             executeEventHandlersIfNeeded(Event.TYPE_LOAD);
290         }
291 
292         try {
293             while (!afterLoadActions_.isEmpty()) {
294                 final PostponedAction action = afterLoadActions_.remove(0);
295                 action.execute();
296             }
297         }
298         catch (final IOException e) {
299             throw e;
300         }
301         catch (final Exception e) {
302             throw new RuntimeException(e);
303         }
304         executeRefreshIfNeeded();
305     }
306 
307     /**
308      * Adds an action that should be executed once the page has been loaded.
309      * @param action the action
310      */
311     void addAfterLoadAction(final PostponedAction action) {
312         afterLoadActions_.add(action);
313     }
314 
315     /**
316      * Clean up this page.
317      */
318     @Override
319     public void cleanUp() {
320         //To avoid endless recursion caused by window.close() in onUnload
321         if (cleaning_) {
322             return;
323         }
324         cleaning_ = true;
325         super.cleanUp();
326         executeEventHandlersIfNeeded(Event.TYPE_UNLOAD);
327         deregisterFramesIfNeeded();
328         cleaning_ = false;
329         if (autoCloseableList_ != null) {
330             for (final AutoCloseable closeable : new ArrayList<>(autoCloseableList_)) {
331                 try {
332                     closeable.close();
333                 }
334                 catch (final Exception e) {
335                     throw new RuntimeException(e);
336                 }
337             }
338         }
339     }
340 
341     /**
342      * {@inheritDoc}
343      */
344     @Override
345     public HtmlElement getDocumentElement() {
346         return (HtmlElement) super.getDocumentElement();
347     }
348 
349     /**
350      * Returns the <tt>body</tt> element (or <tt>frameset</tt> element), or {@code null} if it does not yet exist.
351      * @return the <tt>body</tt> element (or <tt>frameset</tt> element), or {@code null} if it does not yet exist
352      */
353     public HtmlElement getBody() {
354         final HtmlElement doc = getDocumentElement();
355         if (doc != null) {
356             for (final DomNode node : doc.getChildren()) {
357                 if (node instanceof HtmlBody || node instanceof HtmlFrameSet) {
358                     return (HtmlElement) node;
359                 }
360             }
361         }
362         return null;
363     }
364 
365     /**
366      * Returns the head element.
367      * @return the head element
368      */
369     public HtmlElement getHead() {
370         final HtmlElement doc = getDocumentElement();
371         if (doc != null) {
372             for (final DomNode node : doc.getChildren()) {
373                 if (node instanceof HtmlHead) {
374                     return (HtmlElement) node;
375                 }
376             }
377         }
378         return null;
379     }
380 
381     /**
382      * {@inheritDoc}
383      */
384     @Override
385     public Document getOwnerDocument() {
386         return null;
387     }
388 
389     /**
390      * {@inheritDoc}
391      * Not yet implemented.
392      */
393     @Override
394     public org.w3c.dom.Node importNode(final org.w3c.dom.Node importedNode, final boolean deep) {
395         throw new UnsupportedOperationException("HtmlPage.importNode is not yet implemented.");
396     }
397 
398     /**
399      * {@inheritDoc}
400      * Not yet implemented.
401      */
402     @Override
403     public String getInputEncoding() {
404         throw new UnsupportedOperationException("HtmlPage.getInputEncoding is not yet implemented.");
405     }
406 
407     /**
408      * {@inheritDoc}
409      */
410     @Override
411     public String getXmlEncoding() {
412         return null;
413     }
414 
415     /**
416      * {@inheritDoc}
417      */
418     @Override
419     public boolean getXmlStandalone() {
420         return false;
421     }
422 
423     /**
424      * {@inheritDoc}
425      * Not yet implemented.
426      */
427     @Override
428     public void setXmlStandalone(final boolean xmlStandalone) throws DOMException {
429         throw new UnsupportedOperationException("HtmlPage.setXmlStandalone is not yet implemented.");
430     }
431 
432     /**
433      * {@inheritDoc}
434      */
435     @Override
436     public String getXmlVersion() {
437         return null;
438     }
439 
440     /**
441      * {@inheritDoc}
442      * Not yet implemented.
443      */
444     @Override
445     public void setXmlVersion(final String xmlVersion) throws DOMException {
446         throw new UnsupportedOperationException("HtmlPage.setXmlVersion is not yet implemented.");
447     }
448 
449     /**
450      * {@inheritDoc}
451      * Not yet implemented.
452      */
453     @Override
454     public boolean getStrictErrorChecking() {
455         throw new UnsupportedOperationException("HtmlPage.getStrictErrorChecking is not yet implemented.");
456     }
457 
458     /**
459      * {@inheritDoc}
460      * Not yet implemented.
461      */
462     @Override
463     public void setStrictErrorChecking(final boolean strictErrorChecking) {
464         throw new UnsupportedOperationException("HtmlPage.setStrictErrorChecking is not yet implemented.");
465     }
466 
467     /**
468      * {@inheritDoc}
469      * Not yet implemented.
470      */
471     @Override
472     public String getDocumentURI() {
473         throw new UnsupportedOperationException("HtmlPage.getDocumentURI is not yet implemented.");
474     }
475 
476     /**
477      * {@inheritDoc}
478      * Not yet implemented.
479      */
480     @Override
481     public void setDocumentURI(final String documentURI) {
482         throw new UnsupportedOperationException("HtmlPage.setDocumentURI is not yet implemented.");
483     }
484 
485     /**
486      * {@inheritDoc}
487      * Not yet implemented.
488      */
489     @Override
490     public org.w3c.dom.Node adoptNode(final org.w3c.dom.Node source) throws DOMException {
491         throw new UnsupportedOperationException("HtmlPage.adoptNode is not yet implemented.");
492     }
493 
494     /**
495      * {@inheritDoc}
496      * Not yet implemented.
497      */
498     @Override
499     public DOMConfiguration getDomConfig() {
500         throw new UnsupportedOperationException("HtmlPage.getDomConfig is not yet implemented.");
501     }
502 
503     /**
504      * {@inheritDoc}
505      * Not yet implemented.
506      */
507     @Override
508     public org.w3c.dom.Node renameNode(final org.w3c.dom.Node newNode, final String namespaceURI,
509         final String qualifiedName) throws DOMException {
510         throw new UnsupportedOperationException("HtmlPage.renameNode is not yet implemented.");
511     }
512 
513     /**
514      * {@inheritDoc}
515      */
516     @Override
517     public Charset getCharset() {
518         if (originalCharset_ == null) {
519             originalCharset_ = getWebResponse().getContentCharset();
520         }
521         return originalCharset_;
522     }
523 
524     /**
525      * {@inheritDoc}
526      */
527     @Override
528     public String getContentType() {
529         return getWebResponse().getContentType();
530     }
531 
532     /**
533      * {@inheritDoc}
534      * Not yet implemented.
535      */
536     @Override
537     public DOMImplementation getImplementation() {
538         throw new UnsupportedOperationException("HtmlPage.getImplementation is not yet implemented.");
539     }
540 
541     /**
542      * {@inheritDoc}
543      * @param tagName the tag name, preferably in lowercase
544      */
545     @Override
546     public DomElement createElement(String tagName) {
547         if (tagName.indexOf(':') == -1) {
548             tagName = tagName.toLowerCase(Locale.ROOT);
549         }
550         return HTMLParser.getFactory(tagName).createElementNS(this, null, tagName, null, true);
551     }
552 
553     /**
554      * {@inheritDoc}
555      */
556     @Override
557     public DomElement createElementNS(final String namespaceURI, final String qualifiedName) {
558         return HTMLParser.getElementFactory(this, namespaceURI, qualifiedName, true, false)
559             .createElementNS(this, namespaceURI, qualifiedName, null, true);
560     }
561 
562     /**
563      * {@inheritDoc}
564      * Not yet implemented.
565      */
566     @Override
567     public Attr createAttributeNS(final String namespaceURI, final String qualifiedName) {
568         throw new UnsupportedOperationException("HtmlPage.createAttributeNS is not yet implemented.");
569     }
570 
571     /**
572      * {@inheritDoc}
573      * Not yet implemented.
574      */
575     @Override
576     public EntityReference createEntityReference(final String id) {
577         throw new UnsupportedOperationException("HtmlPage.createEntityReference is not yet implemented.");
578     }
579 
580     /**
581      * {@inheritDoc}
582      * Not yet implemented.
583      */
584     @Override
585     public ProcessingInstruction createProcessingInstruction(final String namespaceURI, final String qualifiedName) {
586         throw new UnsupportedOperationException("HtmlPage.createProcessingInstruction is not yet implemented.");
587     }
588 
589     /**
590      * {@inheritDoc}
591      */
592     @Override
593     public DomElement getElementById(final String elementId) {
594         final SortedSet<DomElement> elements = idMap_.get(elementId);
595         if (elements != null) {
596             return elements.first();
597         }
598         return null;
599     }
600 
601     /**
602      * Returns the {@link HtmlAnchor} with the specified name.
603      *
604      * @param name the name to search by
605      * @return the {@link HtmlAnchor} with the specified name
606      * @throws ElementNotFoundException if the anchor could not be found
607      */
608     public HtmlAnchor getAnchorByName(final String name) throws ElementNotFoundException {
609         return getDocumentElement().getOneHtmlElementByAttribute("a", "name", name);
610     }
611 
612     /**
613      * Returns the {@link HtmlAnchor} with the specified href.
614      *
615      * @param href the string to search by
616      * @return the HtmlAnchor
617      * @throws ElementNotFoundException if the anchor could not be found
618      */
619     public HtmlAnchor getAnchorByHref(final String href) throws ElementNotFoundException {
620         return getDocumentElement().getOneHtmlElementByAttribute("a", "href", href);
621     }
622 
623     /**
624      * Returns a list of all anchors contained in this page.
625      * @return the list of {@link HtmlAnchor} in this page
626      */
627     public List<HtmlAnchor> getAnchors() {
628         return getDocumentElement().getElementsByTagNameImpl("a");
629     }
630 
631     /**
632      * Returns the first anchor with the specified text.
633      * @param text the text to search for
634      * @return the first anchor that was found
635      * @throws ElementNotFoundException if no anchors are found with the specified text
636      */
637     public HtmlAnchor getAnchorByText(final String text) throws ElementNotFoundException {
638         WebAssert.notNull("text", text);
639 
640         for (final HtmlAnchor anchor : getAnchors()) {
641             if (text.equals(anchor.asText())) {
642                 return anchor;
643             }
644         }
645         throw new ElementNotFoundException("a", "<text>", text);
646     }
647 
648     /**
649      * Returns the first form that matches the specified name.
650      * @param name the name to search for
651      * @return the first form
652      * @exception ElementNotFoundException If no forms match the specified result.
653      */
654     public HtmlForm getFormByName(final String name) throws ElementNotFoundException {
655         final List<HtmlForm> forms = getDocumentElement().getElementsByAttribute("form", "name", name);
656         if (forms.isEmpty()) {
657             throw new ElementNotFoundException("form", "name", name);
658         }
659         return forms.get(0);
660     }
661 
662     /**
663      * Returns a list of all the forms in this page.
664      * @return all the forms in this page
665      */
666     public List<HtmlForm> getForms() {
667         return getDocumentElement().getElementsByTagNameImpl("form");
668     }
669 
670     /**
671      * Given a relative URL (ie <tt>/foo</tt>), returns a fully-qualified URL based on
672      * the URL that was used to load this page.
673      *
674      * @param relativeUrl the relative URL
675      * @return the fully-qualified URL for the specified relative URL
676      * @exception MalformedURLException if an error occurred when creating a URL object
677      */
678     public URL getFullyQualifiedUrl(String relativeUrl) throws MalformedURLException {
679         final URL baseUrl = getBaseURL();
680 
681         // to handle http: and http:/ in FF (Bug #474)
682         if (hasFeature(URL_MISSING_SLASHES)) {
683             boolean incorrectnessNotified = false;
684             while (relativeUrl.startsWith("http:") && !relativeUrl.startsWith("http://")) {
685                 if (!incorrectnessNotified) {
686                     notifyIncorrectness("Incorrect URL \"" + relativeUrl + "\" has been corrected");
687                     incorrectnessNotified = true;
688                 }
689                 relativeUrl = "http:/" + relativeUrl.substring(5);
690             }
691         }
692 
693         return WebClient.expandUrl(baseUrl, relativeUrl);
694     }
695 
696     /**
697      * Given a target attribute value, resolve the target using a base target for the page.
698      *
699      * @param elementTarget the target specified as an attribute of the element
700      * @return the resolved target to use for the element
701      */
702     public String getResolvedTarget(final String elementTarget) {
703         final String resolvedTarget;
704         if (base_ == null) {
705             resolvedTarget = elementTarget;
706         }
707         else if (elementTarget != null && !elementTarget.isEmpty()) {
708             resolvedTarget = elementTarget;
709         }
710         else {
711             resolvedTarget = base_.getTargetAttribute();
712         }
713         return resolvedTarget;
714     }
715 
716     /**
717      * Returns a list of ids (strings) that correspond to the tabbable elements
718      * in this page. Return them in the same order specified in {@link #getTabbableElements}
719      *
720      * @return the list of id's
721      */
722     public List<String> getTabbableElementIds() {
723         final List<String> list = new ArrayList<>();
724 
725         for (final HtmlElement element : getTabbableElements()) {
726             list.add(element.getId());
727         }
728 
729         return Collections.unmodifiableList(list);
730     }
731 
732     /**
733      * Returns a list of all elements that are tabbable in the order that will
734      * be used for tabbing.<p>
735      *
736      * The rules for determining tab order are as follows:
737      * <ol>
738      *   <li>Those elements that support the tabindex attribute and assign a
739      *   positive value to it are navigated first. Navigation proceeds from the
740      *   element with the lowest tabindex value to the element with the highest
741      *   value. Values need not be sequential nor must they begin with any
742      *   particular value. Elements that have identical tabindex values should
743      *   be navigated in the order they appear in the character stream.
744      *   <li>Those elements that do not support the tabindex attribute or
745      *   support it and assign it a value of "0" are navigated next. These
746      *   elements are navigated in the order they appear in the character
747      *   stream.
748      *   <li>Elements that are disabled do not participate in the tabbing
749      *   order.
750      * </ol>
751      * Additionally, the value of tabindex must be within 0 and 32767. Any
752      * values outside this range will be ignored.<p>
753      *
754      * The following elements support the <tt>tabindex</tt> attribute: A, AREA, BUTTON,
755      * INPUT, OBJECT, SELECT, and TEXTAREA.<p>
756      *
757      * @return all the tabbable elements in proper tab order
758      */
759     public List<HtmlElement> getTabbableElements() {
760         final List<HtmlElement> tabbableElements = new ArrayList<>();
761         for (final HtmlElement element : getHtmlElementDescendants()) {
762             final String tagName = element.getTagName();
763             if (TABBABLE_TAGS.contains(tagName)) {
764                 final boolean disabled = element.hasAttribute("disabled");
765                 if (!disabled && element.getTabIndex() != HtmlElement.TAB_INDEX_OUT_OF_BOUNDS) {
766                     tabbableElements.add(element);
767                 }
768             }
769         }
770         Collections.sort(tabbableElements, createTabOrderComparator());
771         return Collections.unmodifiableList(tabbableElements);
772     }
773 
774     private static Comparator<HtmlElement> createTabOrderComparator() {
775         return new Comparator<HtmlElement>() {
776             @Override
777             public int compare(final HtmlElement element1, final HtmlElement element2) {
778                 final Short i1 = element1.getTabIndex();
779                 final Short i2 = element2.getTabIndex();
780 
781                 final short index1;
782                 if (i1 != null) {
783                     index1 = i1.shortValue();
784                 }
785                 else {
786                     index1 = -1;
787                 }
788 
789                 final short index2;
790                 if (i2 != null) {
791                     index2 = i2.shortValue();
792                 }
793                 else {
794                     index2 = -1;
795                 }
796 
797                 final int result;
798                 if (index1 > 0 && index2 > 0) {
799                     result = index1 - index2;
800                 }
801                 else if (index1 > 0) {
802                     result = -1;
803                 }
804                 else if (index2 > 0) {
805                     result = +1;
806                 }
807                 else if (index1 == index2) {
808                     result = 0;
809                 }
810                 else {
811                     result = index2 - index1;
812                 }
813 
814                 return result;
815             }
816         };
817     }
818 
819     /**
820      * Returns the HTML element that is assigned to the specified access key. An
821      * access key (aka mnemonic key) is used for keyboard navigation of the
822      * page.<p>
823      *
824      * Only the following HTML elements may have <tt>accesskey</tt>s defined: A, AREA,
825      * BUTTON, INPUT, LABEL, LEGEND, and TEXTAREA.
826      *
827      * @param accessKey the key to look for
828      * @return the HTML element that is assigned to the specified key or null
829      *      if no elements can be found that match the specified key.
830      */
831     public HtmlElement getHtmlElementByAccessKey(final char accessKey) {
832         final List<HtmlElement> elements = getHtmlElementsByAccessKey(accessKey);
833         if (elements.isEmpty()) {
834             return null;
835         }
836         return elements.get(0);
837     }
838 
839     /**
840      * Returns all the HTML elements that are assigned to the specified access key. An
841      * access key (aka mnemonic key) is used for keyboard navigation of the
842      * page.<p>
843      *
844      * The HTML specification seems to indicate that one accesskey cannot be used
845      * for multiple elements however Internet Explorer does seem to support this.
846      * It's worth noting that Mozilla does not support multiple elements with one
847      * access key so you are making your HTML browser specific if you rely on this
848      * feature.<p>
849      *
850      * Only the following HTML elements may have <tt>accesskey</tt>s defined: A, AREA,
851      * BUTTON, INPUT, LABEL, LEGEND, and TEXTAREA.
852      *
853      * @param accessKey the key to look for
854      * @return the elements that are assigned to the specified accesskey
855      */
856     public List<HtmlElement> getHtmlElementsByAccessKey(final char accessKey) {
857         final List<HtmlElement> elements = new ArrayList<>();
858 
859         final String searchString = Character.toString(accessKey).toLowerCase(Locale.ROOT);
860         for (final HtmlElement element : getHtmlElementDescendants()) {
861             if (ACCEPTABLE_TAG_NAMES.contains(element.getTagName())) {
862                 final String accessKeyAttribute = element.getAttribute("accesskey");
863                 if (searchString.equalsIgnoreCase(accessKeyAttribute)) {
864                     elements.add(element);
865                 }
866             }
867         }
868 
869         return elements;
870     }
871 
872     /**
873      * <p>Executes the specified JavaScript code within the page. The usage would be similar to what can
874      * be achieved to execute JavaScript in the current page by entering "javascript:...some JS code..."
875      * in the URL field of a native browser.</p>
876      * <p><b>Note:</b> the provided code won't be executed if JavaScript has been disabled on the WebClient
877      * (see {@link com.gargoylesoftware.htmlunit.WebClientOptions#isJavaScriptEnabled()}.</p>
878      * @param sourceCode the JavaScript code to execute
879      * @return a ScriptResult which will contain both the current page (which may be different than
880      * the previous page) and a JavaScript result object
881      */
882     public ScriptResult executeJavaScript(final String sourceCode) {
883         return executeJavaScript(sourceCode, "injected script", 1);
884     }
885 
886     /**
887      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
888      * <p>
889      * Execute the specified JavaScript if a JavaScript engine was successfully
890      * instantiated. If this JavaScript causes the current page to be reloaded
891      * (through location="" or form.submit()) then return the new page. Otherwise
892      * return the current page.
893      * </p>
894      * <p><b>Please note:</b> Although this method is public, it is not intended for
895      * general execution of JavaScript. Users of HtmlUnit should interact with the pages
896      * as a user would by clicking on buttons or links and having the JavaScript event
897      * handlers execute as needed..
898      * </p>
899      *
900      * @param sourceCode the JavaScript code to execute
901      * @param sourceName the name for this chunk of code (will be displayed in error messages)
902      * @param startLine the line at which the script source starts
903      * @return a ScriptResult which will contain both the current page (which may be different than
904      * the previous page and a JavaScript result object.
905      */
906     public ScriptResult executeJavaScript(String sourceCode, final String sourceName, final int startLine) {
907         if (!getWebClient().getOptions().isJavaScriptEnabled()) {
908             return new ScriptResult(null, this);
909         }
910 
911         if (StringUtils.startsWithIgnoreCase(sourceCode, JavaScriptURLConnection.JAVASCRIPT_PREFIX)) {
912             sourceCode = sourceCode.substring(JavaScriptURLConnection.JAVASCRIPT_PREFIX.length()).trim();
913             if (sourceCode.startsWith("return ")) {
914                 sourceCode = sourceCode.substring("return ".length());
915             }
916         }
917 
918         final Object result = getWebClient().getJavaScriptEngine().execute(this, sourceCode, sourceName, startLine);
919         return new ScriptResult(result, getWebClient().getCurrentWindow().getEnclosedPage());
920     }
921 
922     /** Various possible external JavaScript file loading results. */
923     enum JavaScriptLoadResult {
924         /** The load was aborted and nothing was done. */
925         NOOP,
926         /** The external JavaScript file was downloaded and compiled successfully. */
927         SUCCESS,
928         /** The external JavaScript file was not downloaded successfully. */
929         DOWNLOAD_ERROR,
930         /** The external JavaScript file was downloaded but was not compiled successfully. */
931         COMPILATION_ERROR
932     }
933 
934     /**
935      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
936      *
937      * @param srcAttribute the source attribute from the script tag
938      * @param scriptCharset the charset from the script tag
939      * @return the result of loading the specified external JavaScript file
940      * @throws FailingHttpStatusCodeException if the request's status code indicates a request
941      *         failure and the {@link WebClient} was configured to throw exceptions on failing
942      *         HTTP status codes
943      */
944     JavaScriptLoadResult loadExternalJavaScriptFile(final String srcAttribute, final Charset scriptCharset)
945         throws FailingHttpStatusCodeException {
946 
947         final WebClient client = getWebClient();
948         if (StringUtils.isBlank(srcAttribute) || !client.getOptions().isJavaScriptEnabled()) {
949             return JavaScriptLoadResult.NOOP;
950         }
951 
952         final URL scriptURL;
953         try {
954             scriptURL = getFullyQualifiedUrl(srcAttribute);
955             final String protocol = scriptURL.getProtocol();
956             if ("javascript".equals(protocol)) {
957                 LOG.info("Ignoring script src [" + srcAttribute + "]");
958                 return JavaScriptLoadResult.NOOP;
959             }
960             if (!"http".equals(protocol) && !"https".equals(protocol)
961                     && !"data".equals(protocol) && !"file".equals(protocol)) {
962                 client.getJavaScriptErrorListener().malformedScriptURL(this, srcAttribute,
963                         new MalformedURLException("unknown protocol: '" + protocol + "'"));
964                 return JavaScriptLoadResult.NOOP;
965             }
966         }
967         catch (final MalformedURLException e) {
968             client.getJavaScriptErrorListener().malformedScriptURL(this, srcAttribute, e);
969             return JavaScriptLoadResult.NOOP;
970         }
971 
972         final Object script;
973         try {
974             script = loadJavaScriptFromUrl(scriptURL, scriptCharset);
975         }
976         catch (final IOException e) {
977             client.getJavaScriptErrorListener().loadScriptError(this, scriptURL, e);
978             return JavaScriptLoadResult.DOWNLOAD_ERROR;
979         }
980         catch (final FailingHttpStatusCodeException e) {
981             client.getJavaScriptErrorListener().loadScriptError(this, scriptURL, e);
982             throw e;
983         }
984 
985         if (script == null) {
986             return JavaScriptLoadResult.COMPILATION_ERROR;
987         }
988 
989         @SuppressWarnings("unchecked")
990         final AbstractJavaScriptEngine<Object> engine = (AbstractJavaScriptEngine<Object>) client.getJavaScriptEngine();
991         engine.execute(this, script);
992         return JavaScriptLoadResult.SUCCESS;
993     }
994 
995     /**
996      * Loads JavaScript from the specified URL. This method may return {@code null} if
997      * there is a problem loading the code from the specified URL.
998      *
999      * @param url the URL of the script
1000      * @param scriptCharset the charset from the script tag
1001      * @return the content of the file, or {@code null} if we ran into a compile error
1002      * @throws IOException if there is a problem downloading the JavaScript file
1003      * @throws FailingHttpStatusCodeException if the request's status code indicates a request
1004      *         failure and the {@link WebClient} was configured to throw exceptions on failing
1005      *         HTTP status codes
1006      */
1007     private Object loadJavaScriptFromUrl(final URL url, final Charset scriptCharset) throws IOException,
1008         FailingHttpStatusCodeException {
1009 
1010         final WebRequest referringRequest = getWebResponse().getWebRequest();
1011 
1012         final WebClient client = getWebClient();
1013         final Cache cache = client.getCache();
1014 
1015         final WebRequest request = new WebRequest(url, getWebClient().getBrowserVersion().getScriptAcceptHeader());
1016         request.setAdditionalHeaders(new HashMap<>(referringRequest.getAdditionalHeaders()));
1017         request.setAdditionalHeader("Referer", referringRequest.getUrl().toString());
1018         request.setAdditionalHeader("Accept", client.getBrowserVersion().getScriptAcceptHeader());
1019 
1020         // our cache is a bit strange;
1021         // loadWebResponse check the cache for the web response
1022         // AND also fixes the request url for the following cache lookups
1023         final WebResponse response = client.loadWebResponse(request);
1024 
1025         // now we can look into the cache with the fixed request for
1026         // a cached script
1027         final Object cachedScript = cache.getCachedObject(request);
1028         if (cachedScript instanceof Script) {
1029             return cachedScript;
1030         }
1031 
1032         client.printContentIfNecessary(response);
1033         client.throwFailingHttpStatusCodeExceptionIfNecessary(response);
1034 
1035         final int statusCode = response.getStatusCode();
1036         final boolean successful = statusCode >= HttpStatus.SC_OK && statusCode < HttpStatus.SC_MULTIPLE_CHOICES;
1037         final boolean noContent = statusCode == HttpStatus.SC_NO_CONTENT;
1038         if (!successful || noContent) {
1039             throw new IOException("Unable to download JavaScript from '" + url + "' (status " + statusCode + ").");
1040         }
1041 
1042         //http://www.ietf.org/rfc/rfc4329.txt
1043         final String contentType = response.getContentType();
1044         if (!"application/javascript".equalsIgnoreCase(contentType)
1045             && !"application/ecmascript".equalsIgnoreCase(contentType)) {
1046             // warn about obsolete or not supported content types
1047             if ("text/javascript".equals(contentType)
1048                     || "text/ecmascript".equals(contentType)
1049                     || "application/x-javascript".equalsIgnoreCase(contentType)) {
1050                 getWebClient().getIncorrectnessListener().notify(
1051                     "Obsolete content type encountered: '" + contentType + "'.", this);
1052             }
1053             else {
1054                 getWebClient().getIncorrectnessListener().notify(
1055                         "Expected content type of 'application/javascript' or 'application/ecmascript' for "
1056                         + "remotely loaded JavaScript element at '" + url + "', "
1057                         + "but got '" + contentType + "'.", this);
1058             }
1059         }
1060 
1061         Charset scriptEncoding = Charset.forName("windows-1252");
1062         final Charset contentCharset = EncodingSniffer.sniffEncodingFromHttpHeaders(response.getResponseHeaders());
1063         if (contentCharset == null) {
1064             // use info from script tag or fall back to utf-8
1065             if (scriptCharset != null && scriptCharset != ISO_8859_1) {
1066                 scriptEncoding = scriptCharset;
1067             }
1068         }
1069         else if (contentCharset != ISO_8859_1) {
1070             scriptEncoding = contentCharset;
1071         }
1072 
1073         final String scriptCode = response.getContentAsString(scriptEncoding);
1074         if (null != scriptCode) {
1075             final AbstractJavaScriptEngine<?> javaScriptEngine = client.getJavaScriptEngine();
1076             final Object script = javaScriptEngine.compile(this, scriptCode, url.toExternalForm(), 1);
1077             if (script != null && cache.cacheIfPossible(request, response, script)) {
1078                 // no cleanup if the response is stored inside the cache
1079                 return script;
1080             }
1081 
1082             response.cleanUp();
1083             return script;
1084         }
1085 
1086         response.cleanUp();
1087         return null;
1088     }
1089 
1090     /**
1091      * Returns the title of this page or an empty string if the title wasn't specified.
1092      *
1093      * @return the title of this page or an empty string if the title wasn't specified
1094      */
1095     public String getTitleText() {
1096         final HtmlTitle titleElement = getTitleElement();
1097         if (titleElement != null) {
1098             return titleElement.asText();
1099         }
1100         return "";
1101     }
1102 
1103     /**
1104      * Sets the text for the title of this page. If there is not a title element
1105      * on this page, then one has to be generated.
1106      * @param message the new text
1107      */
1108     public void setTitleText(final String message) {
1109         HtmlTitle titleElement = getTitleElement();
1110         if (titleElement == null) {
1111             if (LOG.isDebugEnabled()) {
1112                 LOG.debug("No title element, creating one");
1113             }
1114             final HtmlHead head = (HtmlHead) getFirstChildElement(getDocumentElement(), HtmlHead.class);
1115             if (head == null) {
1116                 // perhaps should we create head too?
1117                 throw new IllegalStateException("Headelement was not defined for this page");
1118             }
1119             final Map<String, DomAttr> emptyMap = Collections.emptyMap();
1120             titleElement = new HtmlTitle(HtmlTitle.TAG_NAME, this, emptyMap);
1121             if (head.getFirstChild() != null) {
1122                 head.getFirstChild().insertBefore(titleElement);
1123             }
1124             else {
1125                 head.appendChild(titleElement);
1126             }
1127         }
1128 
1129         titleElement.setNodeValue(message);
1130     }
1131 
1132     /**
1133      * Gets the first child of startElement that is an instance of the given class.
1134      * @param startElement the parent element
1135      * @param clazz the class to search for
1136      * @return {@code null} if no child found
1137      */
1138     private static DomElement getFirstChildElement(final DomElement startElement, final Class<?> clazz) {
1139         if (startElement == null) {
1140             return null;
1141         }
1142         for (final DomElement element : startElement.getChildElements()) {
1143             if (clazz.isInstance(element)) {
1144                 return element;
1145             }
1146         }
1147 
1148         return null;
1149     }
1150 
1151     /**
1152      * Gets the first child of startElement or it's children that is an instance of the given class.
1153      * @param startElement the parent element
1154      * @param clazz the class to search for
1155      * @return {@code null} if no child found
1156      */
1157     private DomElement getFirstChildElementRecursive(final DomElement startElement, final Class<?> clazz) {
1158         if (startElement == null) {
1159             return null;
1160         }
1161         for (final DomElement element : startElement.getChildElements()) {
1162             if (clazz.isInstance(element)) {
1163                 return element;
1164             }
1165             final DomElement childFound = getFirstChildElementRecursive(element, clazz);
1166             if (childFound != null) {
1167                 return childFound;
1168             }
1169         }
1170 
1171         return null;
1172     }
1173 
1174     /**
1175      * Gets the title element for this page. Returns null if one is not found.
1176      *
1177      * @return the title element for this page or null if this is not one
1178      */
1179     private HtmlTitle getTitleElement() {
1180         return (HtmlTitle) getFirstChildElementRecursive(getDocumentElement(), HtmlTitle.class);
1181     }
1182 
1183     /**
1184      * Looks for and executes any appropriate event handlers. Looks for body and frame tags.
1185      * @param eventType either {@link Event#TYPE_LOAD}, {@link Event#TYPE_UNLOAD}, or {@link Event#TYPE_BEFORE_UNLOAD}
1186      * @return {@code true} if user accepted <tt>onbeforeunload</tt> (not relevant to other events)
1187      */
1188     private boolean executeEventHandlersIfNeeded(final String eventType) {
1189         // If JavaScript isn't enabled, there's nothing for us to do.
1190         if (!getWebClient().getOptions().isJavaScriptEnabled()) {
1191             return true;
1192         }
1193 
1194         // Execute the specified event on the document element.
1195         final WebWindow window = getEnclosingWindow();
1196         if (window.getScriptableObject() instanceof Window) {
1197             final HtmlElement element = getDocumentElement();
1198             if (element == null) { // happens for instance if document.documentElement has been removed from parent
1199                 return true;
1200             }
1201             final Event event;
1202             if (eventType.equals(Event.TYPE_BEFORE_UNLOAD)) {
1203                 event = new BeforeUnloadEvent(element, eventType);
1204             }
1205             else {
1206                 event = new Event(element, eventType);
1207             }
1208             final ScriptResult result = element.fireEvent(event);
1209             if (!isOnbeforeunloadAccepted(this, event, result)) {
1210                 return false;
1211             }
1212         }
1213 
1214         // If this page was loaded in a frame, execute the version of the event specified on the frame tag.
1215         if (window instanceof FrameWindow) {
1216             final FrameWindow fw = (FrameWindow) window;
1217             final BaseFrameElement frame = fw.getFrameElement();
1218 
1219             // if part of an document fragment, then the load event is not triggered
1220             if (Event.TYPE_LOAD.equals(eventType) && frame.getParentNode() instanceof DomDocumentFragment) {
1221                 return true;
1222             }
1223 
1224             if (frame.hasEventHandlers("on" + eventType)) {
1225                 if (LOG.isDebugEnabled()) {
1226                     LOG.debug("Executing on" + eventType + " handler for " + frame);
1227                 }
1228                 if (window.getScriptableObject() instanceof Window) {
1229                     final Event event;
1230                     if (eventType.equals(Event.TYPE_BEFORE_UNLOAD)) {
1231                         event = new BeforeUnloadEvent(frame, eventType);
1232                     }
1233                     else {
1234                         event = new Event(frame, eventType);
1235                     }
1236                     final ScriptResult result = ((Node) frame.getScriptableObject()).executeEventLocally(event);
1237                     if (!isOnbeforeunloadAccepted((HtmlPage) frame.getPage(), event, result)) {
1238                         return false;
1239                     }
1240                 }
1241             }
1242         }
1243 
1244         return true;
1245     }
1246 
1247     /**
1248      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
1249      *
1250      * @return true if the OnbeforeunloadHandler has accepted to change the page
1251      */
1252     public boolean isOnbeforeunloadAccepted() {
1253         return executeEventHandlersIfNeeded(Event.TYPE_BEFORE_UNLOAD);
1254     }
1255 
1256     private boolean isOnbeforeunloadAccepted(final HtmlPage page, final Event event, final ScriptResult result) {
1257         if (event.getType().equals(Event.TYPE_BEFORE_UNLOAD)) {
1258             final boolean ie = hasFeature(JS_CALL_RESULT_IS_LAST_RETURN_VALUE);
1259             final String message = getBeforeUnloadMessage(event, result, ie);
1260             if (message != null) {
1261                 final OnbeforeunloadHandler handler = getWebClient().getOnbeforeunloadHandler();
1262                 if (handler == null) {
1263                     LOG.warn("document.onbeforeunload() returned a string in event.returnValue,"
1264                             + " but no onbeforeunload handler installed.");
1265                 }
1266                 else {
1267                     return handler.handleEvent(page, message);
1268                 }
1269             }
1270         }
1271         return true;
1272     }
1273 
1274     private static String getBeforeUnloadMessage(final Event event, final ScriptResult result, final boolean ie) {
1275         String message = null;
1276         if (event.getReturnValue() != Undefined.instance) {
1277             if (!ie || event.getReturnValue() != null || result == null || result.getJavaScriptResult() == null
1278                     || result.getJavaScriptResult() == Undefined.instance) {
1279                 message = Context.toString(event.getReturnValue());
1280             }
1281         }
1282         else {
1283             if (result != null) {
1284                 if (ie) {
1285                     if (result.getJavaScriptResult() != Undefined.instance) {
1286                         message = Context.toString(result.getJavaScriptResult());
1287                     }
1288                 }
1289                 else if (result.getJavaScriptResult() != null
1290                         && result.getJavaScriptResult() != Undefined.instance) {
1291                     message = Context.toString(result.getJavaScriptResult());
1292                 }
1293             }
1294         }
1295         return message;
1296     }
1297 
1298     /**
1299      * If a refresh has been specified either through a meta tag or an HTTP
1300      * response header, then perform that refresh.
1301      * @throws IOException if an IO problem occurs
1302      */
1303     private void executeRefreshIfNeeded() throws IOException {
1304         // If this page is not in a frame then a refresh has already happened,
1305         // most likely through the JavaScript onload handler, so we don't do a
1306         // second refresh.
1307         final WebWindow window = getEnclosingWindow();
1308         if (window == null) {
1309             return;
1310         }
1311 
1312         final String refreshString = getRefreshStringOrNull();
1313         if (refreshString == null || refreshString.isEmpty()) {
1314             return;
1315         }
1316 
1317         final double time;
1318         final URL url;
1319 
1320         int index = StringUtils.indexOfAnyBut(refreshString, "0123456789");
1321         final boolean timeOnly = index == -1;
1322 
1323         if (timeOnly) {
1324             // Format: <meta http-equiv='refresh' content='10'>
1325             try {
1326                 time = Double.parseDouble(refreshString);
1327             }
1328             catch (final NumberFormatException e) {
1329                 LOG.error("Malformed refresh string (no ';' but not a number): " + refreshString, e);
1330                 return;
1331             }
1332             url = getUrl();
1333         }
1334         else {
1335             // Format: <meta http-equiv='refresh' content='10;url=http://www.blah.com'>
1336             try {
1337                 time = Double.parseDouble(refreshString.substring(0, index).trim());
1338             }
1339             catch (final NumberFormatException e) {
1340                 LOG.error("Malformed refresh string (no valid number before ';') " + refreshString, e);
1341                 return;
1342             }
1343             index = refreshString.toLowerCase(Locale.ROOT).indexOf("url=", index);
1344             if (index == -1) {
1345                 LOG.error("Malformed refresh string (found ';' but no 'url='): " + refreshString);
1346                 return;
1347             }
1348             final StringBuilder builder = new StringBuilder(refreshString.substring(index + 4));
1349             if (StringUtils.isBlank(builder.toString())) {
1350                 //content='10; URL=' is treated as content='10'
1351                 url = getUrl();
1352             }
1353             else {
1354                 if (builder.charAt(0) == '"' || builder.charAt(0) == 0x27) {
1355                     builder.deleteCharAt(0);
1356                 }
1357                 if (builder.charAt(builder.length() - 1) == '"' || builder.charAt(builder.length() - 1) == 0x27) {
1358                     builder.deleteCharAt(builder.length() - 1);
1359                 }
1360                 final String urlString = builder.toString();
1361                 try {
1362                     url = getFullyQualifiedUrl(urlString);
1363                 }
1364                 catch (final MalformedURLException e) {
1365                     LOG.error("Malformed URL in refresh string: " + refreshString, e);
1366                     throw e;
1367                 }
1368             }
1369         }
1370 
1371         final int timeRounded = (int) time;
1372         checkRecursion();
1373         getWebClient().getRefreshHandler().handleRefresh(this, url, timeRounded);
1374     }
1375 
1376     private void checkRecursion() {
1377         final StackTraceElement[] elements = new Exception().getStackTrace();
1378         if (elements.length > 500) {
1379             for (int i = 0; i < 500; i++) {
1380                 if (!elements[i].getClassName().startsWith("com.gargoylesoftware.htmlunit.")) {
1381                     return;
1382                 }
1383             }
1384             final WebResponse webResponse = getWebResponse();
1385             throw new FailingHttpStatusCodeException("Too much redirect for "
1386                     + webResponse.getWebRequest().getUrl(), webResponse);
1387         }
1388     }
1389 
1390     /**
1391      * Returns an auto-refresh string if specified. This will look in both the meta
1392      * tags and inside the HTTP response headers.
1393      * @return the auto-refresh string
1394      */
1395     private String getRefreshStringOrNull() {
1396         final List<HtmlMeta> metaTags = getMetaTags("refresh");
1397         if (!metaTags.isEmpty()) {
1398             return metaTags.get(0).getContentAttribute().trim();
1399         }
1400         return getWebResponse().getResponseHeaderValue("Refresh");
1401     }
1402 
1403     /**
1404      * Executes any deferred scripts, if necessary.
1405      */
1406     private void executeDeferredScriptsIfNeeded() {
1407         if (!getWebClient().getOptions().isJavaScriptEnabled()) {
1408             return;
1409         }
1410         if (hasFeature(JS_DEFERRED)) {
1411             final HtmlElement doc = getDocumentElement();
1412             final List<HtmlElement> elements = doc.getElementsByTagName("script");
1413             for (final HtmlElement e : elements) {
1414                 if (e instanceof HtmlScript) {
1415                     final HtmlScript script = (HtmlScript) e;
1416                     if (script.isDeferred()) {
1417                         script.executeScriptIfNeeded();
1418                     }
1419                 }
1420             }
1421         }
1422     }
1423 
1424     /**
1425      * Sets the ready state on any deferred scripts, if necessary.
1426      */
1427     private void setReadyStateOnDeferredScriptsIfNeeded() {
1428         if (getWebClient().getOptions().isJavaScriptEnabled() && hasFeature(JS_DEFERRED)) {
1429             final List<HtmlElement> elements = getDocumentElement().getElementsByTagName("script");
1430             for (final HtmlElement e : elements) {
1431                 if (e instanceof HtmlScript) {
1432                     final HtmlScript script = (HtmlScript) e;
1433                     if (script.isDeferred()) {
1434                         script.setAndExecuteReadyState(READY_STATE_COMPLETE);
1435                     }
1436                 }
1437             }
1438         }
1439     }
1440 
1441     /**
1442      * Deregister frames that are no longer in use.
1443      */
1444     public void deregisterFramesIfNeeded() {
1445         for (final WebWindow window : getFrames()) {
1446             getWebClient().deregisterWebWindow(window);
1447             final Page page = window.getEnclosedPage();
1448             if (page != null && page.isHtmlPage()) {
1449                 // seems quite silly, but for instance if the src attribute of an iframe is not
1450                 // set, the error only occurs when leaving the page
1451                 ((HtmlPage) page).deregisterFramesIfNeeded();
1452             }
1453         }
1454     }
1455 
1456     /**
1457      * Returns a list containing all the frames (from frame and iframe tags) in this page.
1458      * @return a list of {@link FrameWindow}
1459      */
1460     public List<FrameWindow> getFrames() {
1461         final List<FrameWindow> list = new ArrayList<>(frameElements_.size());
1462         for (final BaseFrameElement frameElement : frameElements_) {
1463             list.add(frameElement.getEnclosedWindow());
1464         }
1465         return list;
1466     }
1467 
1468     /**
1469      * Returns the first frame contained in this page with the specified name.
1470      * @param name the name to search for
1471      * @return the first frame found
1472      * @exception ElementNotFoundException If no frame exist in this page with the specified name.
1473      */
1474     public FrameWindow getFrameByName(final String name) throws ElementNotFoundException {
1475         for (final FrameWindow frame : getFrames()) {
1476             if (frame.getName().equals(name)) {
1477                 return frame;
1478             }
1479         }
1480 
1481         throw new ElementNotFoundException("frame or iframe", "name", name);
1482     }
1483 
1484     /**
1485      * Simulate pressing an access key. This may change the focus, may click buttons and may invoke
1486      * JavaScript.
1487      *
1488      * @param accessKey the key that will be pressed
1489      * @return the element that has the focus after pressing this access key or null if no element
1490      * has the focus.
1491      * @throws IOException if an IO error occurs during the processing of this access key (this
1492      *         would only happen if the access key triggered a button which in turn caused a page load)
1493      */
1494     public DomElement pressAccessKey(final char accessKey) throws IOException {
1495         final HtmlElement element = getHtmlElementByAccessKey(accessKey);
1496         if (element != null) {
1497             element.focus();
1498             final Page newPage;
1499             if (element instanceof HtmlAnchor || element instanceof HtmlArea || element instanceof HtmlButton
1500                     || element instanceof HtmlInput || element instanceof HtmlLabel || element instanceof HtmlLegend
1501                     || element instanceof HtmlTextArea || element instanceof HtmlArea) {
1502                 newPage = element.click();
1503             }
1504             else {
1505                 newPage = this;
1506             }
1507 
1508             if (newPage != this && getFocusedElement() == element) {
1509                 // The page was reloaded therefore no element on this page will have the focus.
1510                 getFocusedElement().blur();
1511             }
1512         }
1513 
1514         return getFocusedElement();
1515     }
1516 
1517     /**
1518      * Move the focus to the next element in the tab order. To determine the specified tab
1519      * order, refer to {@link HtmlPage#getTabbableElements()}
1520      *
1521      * @return the element that has focus after calling this method
1522      */
1523     public HtmlElement tabToNextElement() {
1524         final List<HtmlElement> elements = getTabbableElements();
1525         if (elements.isEmpty()) {
1526             setFocusedElement(null);
1527             return null;
1528         }
1529 
1530         final HtmlElement elementToGiveFocus;
1531         final DomElement elementWithFocus = getFocusedElement();
1532         if (elementWithFocus == null) {
1533             elementToGiveFocus = elements.get(0);
1534         }
1535         else {
1536             final int index = elements.indexOf(elementWithFocus);
1537             if (index == -1) {
1538                 // The element with focus isn't on this page
1539                 elementToGiveFocus = elements.get(0);
1540             }
1541             else {
1542                 if (index == elements.size() - 1) {
1543                     elementToGiveFocus = elements.get(0);
1544                 }
1545                 else {
1546                     elementToGiveFocus = elements.get(index + 1);
1547                 }
1548             }
1549         }
1550 
1551         setFocusedElement(elementToGiveFocus);
1552         return elementToGiveFocus;
1553     }
1554 
1555     /**
1556      * Move the focus to the previous element in the tab order. To determine the specified tab
1557      * order, refer to {@link HtmlPage#getTabbableElements()}
1558      *
1559      * @return the element that has focus after calling this method
1560      */
1561     public HtmlElement tabToPreviousElement() {
1562         final List<HtmlElement> elements = getTabbableElements();
1563         if (elements.isEmpty()) {
1564             setFocusedElement(null);
1565             return null;
1566         }
1567 
1568         final HtmlElement elementToGiveFocus;
1569         final DomElement elementWithFocus = getFocusedElement();
1570         if (elementWithFocus == null) {
1571             elementToGiveFocus = elements.get(elements.size() - 1);
1572         }
1573         else {
1574             final int index = elements.indexOf(elementWithFocus);
1575             if (index == -1) {
1576                 // The element with focus isn't on this page
1577                 elementToGiveFocus = elements.get(elements.size() - 1);
1578             }
1579             else {
1580                 if (index == 0) {
1581                     elementToGiveFocus = elements.get(elements.size() - 1);
1582                 }
1583                 else {
1584                     elementToGiveFocus = elements.get(index - 1);
1585                 }
1586             }
1587         }
1588 
1589         setFocusedElement(elementToGiveFocus);
1590         return elementToGiveFocus;
1591     }
1592 
1593     /**
1594      * Returns the HTML element with the specified ID. If more than one element
1595      * has this ID (not allowed by the HTML spec), then this method returns the
1596      * first one.
1597      *
1598      * @param elementId the ID value to search for
1599      * @param <E> the element type
1600      * @return the HTML element with the specified ID
1601      * @throws ElementNotFoundException if no element was found matching the specified ID
1602      */
1603     @SuppressWarnings("unchecked")
1604     public <E extends HtmlElement> E getHtmlElementById(final String elementId) throws ElementNotFoundException {
1605         final DomElement element = getElementById(elementId);
1606         if (element == null) {
1607             throw new ElementNotFoundException("*", "id", elementId);
1608         }
1609         return (E) element;
1610     }
1611 
1612     /**
1613      * Returns the elements with the specified ID. If there are no elements
1614      * with the specified ID, this method returns an empty list. Please note that
1615      * the lists returned by this method are immutable.
1616      *
1617      * @param elementId the ID value to search for
1618      * @return the elements with the specified name attribute
1619      */
1620     public List<DomElement> getElementsById(final String elementId) {
1621         final SortedSet<DomElement> elements = idMap_.get(elementId);
1622         if (elements != null) {
1623             return new ArrayList<>(elements);
1624         }
1625         return Collections.emptyList();
1626     }
1627 
1628     /**
1629      * Returns the element with the specified name. If more than one element
1630      * has this name, then this method returns the first one.
1631      *
1632      * @param name the name value to search for
1633      * @param <E> the element type
1634      * @return the element with the specified name
1635      * @throws ElementNotFoundException if no element was found matching the specified name
1636      */
1637     @SuppressWarnings("unchecked")
1638     public <E extends DomElement> E getElementByName(final String name) throws ElementNotFoundException {
1639         final SortedSet<DomElement> elements = nameMap_.get(name);
1640         if (elements != null) {
1641             return (E) elements.first();
1642         }
1643         throw new ElementNotFoundException("*", "name", name);
1644     }
1645 
1646     /**
1647      * Returns the elements with the specified name attribute. If there are no elements
1648      * with the specified name, this method returns an empty list. Please note that
1649      * the lists returned by this method are immutable.
1650      *
1651      * @param name the name value to search for
1652      * @return the elements with the specified name attribute
1653      */
1654     public List<DomElement> getElementsByName(final String name) {
1655         final SortedSet<DomElement> elements = nameMap_.get(name);
1656         if (elements != null) {
1657             return new ArrayList<>(elements);
1658         }
1659         return Collections.emptyList();
1660     }
1661 
1662     /**
1663      * Returns the elements with the specified string for their name or ID. If there are
1664      * no elements with the specified name or ID, this method returns an empty list.
1665      *
1666      * @param idAndOrName the value to search for
1667      * @return the elements with the specified string for their name or ID
1668      */
1669     public List<DomElement> getElementsByIdAndOrName(final String idAndOrName) {
1670         final Collection<DomElement> list1 = idMap_.get(idAndOrName);
1671         final Collection<DomElement> list2 = nameMap_.get(idAndOrName);
1672         final List<DomElement> list = new ArrayList<>();
1673         if (list1 != null) {
1674             list.addAll(list1);
1675         }
1676         if (list2 != null) {
1677             for (final DomElement elt : list2) {
1678                 if (!list.contains(elt)) {
1679                     list.add(elt);
1680                 }
1681             }
1682         }
1683         return list;
1684     }
1685 
1686     /**
1687      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
1688      *
1689      * @param node the node that has just been added to the document
1690      */
1691     void notifyNodeAdded(final DomNode node) {
1692         if (node instanceof DomElement) {
1693             addMappedElement((DomElement) node, true);
1694 
1695             if (node instanceof BaseFrameElement) {
1696                 frameElements_.add((BaseFrameElement) node);
1697             }
1698             for (final HtmlElement child : node.getHtmlElementDescendants()) {
1699                 if (child instanceof BaseFrameElement) {
1700                     frameElements_.add((BaseFrameElement) child);
1701                 }
1702             }
1703 
1704             if ("base".equals(node.getNodeName())) {
1705                 calculateBase();
1706             }
1707         }
1708         node.onAddedToPage();
1709     }
1710 
1711     /**
1712      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
1713      *
1714      * @param node the node that has just been removed from the tree
1715      */
1716     void notifyNodeRemoved(final DomNode node) {
1717         if (node instanceof HtmlElement) {
1718             removeMappedElement((HtmlElement) node, true, true);
1719 
1720             if (node instanceof BaseFrameElement) {
1721                 frameElements_.remove(node);
1722             }
1723             for (final HtmlElement child : node.getHtmlElementDescendants()) {
1724                 if (child instanceof BaseFrameElement) {
1725                     frameElements_.remove(child);
1726                 }
1727             }
1728 
1729             if ("base".equals(node.getNodeName())) {
1730                 calculateBase();
1731             }
1732         }
1733     }
1734 
1735     /**
1736      * Adds an element to the ID and name maps, if necessary.
1737      * @param element the element to be added to the ID and name maps
1738      */
1739     void addMappedElement(final DomElement element) {
1740         addMappedElement(element, false);
1741     }
1742 
1743     /**
1744      * Adds an element to the ID and name maps, if necessary.
1745      * @param element the element to be added to the ID and name maps
1746      * @param recurse indicates if children must be added too
1747      */
1748     void addMappedElement(final DomElement element, final boolean recurse) {
1749         if (isAncestorOf(element)) {
1750             addElement(idMap_, element, "id", recurse);
1751             addElement(nameMap_, element, "name", recurse);
1752         }
1753     }
1754 
1755     private void addElement(final Map<String, SortedSet<DomElement>> map, final DomElement element,
1756             final String attribute, final boolean recurse) {
1757         final String value = getAttributeValue(element, attribute);
1758 
1759         if (DomElement.ATTRIBUTE_NOT_DEFINED != value) {
1760             SortedSet<DomElement> elements = map.get(value);
1761             if (elements == null) {
1762                 elements = new TreeSet<>(documentPositionComparator);
1763                 elements.add(element);
1764                 map.put(value, elements);
1765             }
1766             else if (!elements.contains(element)) {
1767                 elements.add(element);
1768             }
1769         }
1770         if (recurse) {
1771             for (final DomElement child : element.getChildElements()) {
1772                 addElement(map, child, attribute, true);
1773             }
1774         }
1775     }
1776 
1777     private static String getAttributeValue(final DomElement element, final String attribute) {
1778         // first try real attributes
1779         String value = element.getAttribute(attribute);
1780 
1781         if (DomElement.ATTRIBUTE_NOT_DEFINED == value
1782                 && !(element instanceof HtmlApplet)
1783                 && !(element instanceof HtmlObject)) {
1784             // second try are JavaScript attributes
1785             // ...but applets/objects are a bit special so ignore them
1786             final Object o = element.getScriptableObject();
1787             if (o instanceof ScriptableObject) {
1788                 final ScriptableObject scriptObject = (ScriptableObject) o;
1789                 // we have to make sure the scriptObject has a slot for the given attribute.
1790                 // just using get() may use e.g. getWithPreemption().
1791                 if (scriptObject.has(attribute, scriptObject)) {
1792                     final Object jsValue = scriptObject.get(attribute, scriptObject);
1793                     if (jsValue != null && jsValue != Scriptable.NOT_FOUND && jsValue instanceof String) {
1794                         value = (String) jsValue;
1795                     }
1796                 }
1797             }
1798         }
1799         return value;
1800     }
1801 
1802     /**
1803      * Removes an element from the ID and name maps, if necessary.
1804      * @param element the element to be removed from the ID and name maps
1805      */
1806     void removeMappedElement(final HtmlElement element) {
1807         removeMappedElement(element, false, false);
1808     }
1809 
1810     /**
1811      * Removes an element and optionally its children from the ID and name maps, if necessary.
1812      * @param element the element to be removed from the ID and name maps
1813      * @param recurse indicates if children must be removed too
1814      * @param descendant indicates of the element was descendant of this HtmlPage, but now its parent might be null
1815      */
1816     void removeMappedElement(final DomElement element, final boolean recurse, final boolean descendant) {
1817         if (descendant || isAncestorOf(element)) {
1818             removeElement(idMap_, element, "id", recurse);
1819             removeElement(nameMap_, element, "name", recurse);
1820         }
1821     }
1822 
1823     private void removeElement(final Map<String, SortedSet<DomElement>> map, final DomElement element,
1824             final String attribute, final boolean recurse) {
1825         final String value = getAttributeValue(element, attribute);
1826 
1827         if (DomElement.ATTRIBUTE_NOT_DEFINED != value) {
1828             final SortedSet<DomElement> elements = map.remove(value);
1829             if (elements != null && (elements.size() != 1 || !elements.contains(element))) {
1830                 elements.remove(element);
1831                 map.put(value, elements);
1832             }
1833         }
1834         if (recurse) {
1835             for (final DomElement child : element.getChildElements()) {
1836                 removeElement(map, child, attribute, true);
1837             }
1838         }
1839     }
1840 
1841     /**
1842      * Indicates if the attribute name indicates that the owning element is mapped.
1843      * @param document the owning document
1844      * @param attributeName the name of the attribute to consider
1845      * @return {@code true} if the owning element should be mapped in its owning page
1846      */
1847     static boolean isMappedElement(final Document document, final String attributeName) {
1848         return document instanceof HtmlPage
1849             && ("name".equals(attributeName) || "id".equals(attributeName));
1850     }
1851 
1852     private void calculateBase() {
1853         final List<HtmlElement> baseElements = getDocumentElement().getElementsByTagName("base");
1854         switch (baseElements.size()) {
1855             case 0:
1856                 base_ = null;
1857                 break;
1858 
1859             case 1:
1860                 base_ = (HtmlBase) baseElements.get(0);
1861                 break;
1862 
1863             default:
1864                 base_ = (HtmlBase) baseElements.get(0);
1865                 notifyIncorrectness("Multiple 'base' detected, only the first is used.");
1866         }
1867     }
1868 
1869     /**
1870      * Loads the content of the contained frames. This is done after the page is completely loaded, to allow script
1871      * contained in the frames to reference elements from the page located after the closing &lt;/frame&gt; tag.
1872      * @throws FailingHttpStatusCodeException if the server returns a failing status code AND the property
1873      *         {@link WebClient#setThrowExceptionOnFailingStatusCode(boolean)} is set to {@code true}
1874      */
1875     void loadFrames() throws FailingHttpStatusCodeException {
1876         for (final FrameWindow w : getFrames()) {
1877             final BaseFrameElement frame = w.getFrameElement();
1878             // test if the frame should really be loaded:
1879             // if a script has already changed its content, it should be skipped
1880             // use == and not equals(...) to identify initial content (versus URL set to "about:blank")
1881             if (frame.getEnclosedWindow() != null
1882                     && WebClient.URL_ABOUT_BLANK == frame.getEnclosedPage().getUrl()
1883                     && !frame.isContentLoaded()) {
1884                 frame.loadInnerPage();
1885             }
1886         }
1887     }
1888 
1889     /**
1890      * Gives a basic representation for debugging purposes.
1891      * @return a basic representation
1892      */
1893     @Override
1894     public String toString() {
1895         final StringBuilder builder = new StringBuilder();
1896         builder.append("HtmlPage(");
1897         builder.append(getUrl());
1898         builder.append(")@");
1899         builder.append(hashCode());
1900         return builder.toString();
1901     }
1902 
1903     /**
1904      * Gets the meta tag for a given {@code http-equiv} value.
1905      * @param httpEquiv the {@code http-equiv} value
1906      * @return a list of {@link HtmlMeta}
1907      */
1908     protected List<HtmlMeta> getMetaTags(final String httpEquiv) {
1909         if (getDocumentElement() == null) {
1910             return Collections.emptyList(); // weird case, for instance if document.documentElement has been removed
1911         }
1912         final String nameLC = httpEquiv.toLowerCase(Locale.ROOT);
1913         final List<HtmlMeta> tags = getDocumentElement().getElementsByTagNameImpl("meta");
1914         final List<HtmlMeta> foundTags = new ArrayList<>();
1915         for (HtmlMeta htmlMeta : tags) {
1916             if (nameLC.equals(htmlMeta.getHttpEquivAttribute().toLowerCase(Locale.ROOT))) {
1917                 foundTags.add(htmlMeta);
1918             }
1919         }
1920         return foundTags;
1921     }
1922 
1923     /**
1924      * Creates a clone of this instance, and clears cached state to be not shared with the original.
1925      *
1926      * @return a clone of this instance
1927      */
1928     @Override
1929     protected HtmlPage clone() {
1930         final HtmlPage result = (HtmlPage) super.clone();
1931         result.elementWithFocus_ = null;
1932 
1933         result.idMap_ = Collections.synchronizedMap(new HashMap<String, SortedSet<DomElement>>());
1934         result.nameMap_ = Collections.synchronizedMap(new HashMap<String, SortedSet<DomElement>>());
1935 
1936         return result;
1937     }
1938 
1939     /**
1940      * {@inheritDoc}
1941      */
1942     @Override
1943     public HtmlPage cloneNode(final boolean deep) {
1944         // we need the ScriptObject clone before cloning the kids.
1945         final HtmlPage result = (HtmlPage) super.cloneNode(false);
1946         final SimpleScriptable jsObjClone = ((SimpleScriptable) getScriptableObject()).clone();
1947         jsObjClone.setDomNode(result);
1948 
1949         // if deep, clone the kids too, and re initialize parts of the clone
1950         if (deep) {
1951             synchronized (lock_) {
1952                 result.attributeListeners_ = null;
1953             }
1954             result.selectionRanges_ = new ArrayList<>(3);
1955             result.afterLoadActions_ = new ArrayList<>();
1956             result.frameElements_ = new TreeSet<>(documentPositionComparator);
1957             for (DomNode child = getFirstChild(); child != null; child = child.getNextSibling()) {
1958                 result.appendChild(child.cloneNode(true));
1959             }
1960         }
1961         return result;
1962     }
1963 
1964     /**
1965      * Adds an HtmlAttributeChangeListener to the listener list.
1966      * The listener is registered for all attributes of all HtmlElements contained in this page.
1967      *
1968      * @param listener the attribute change listener to be added
1969      * @see #removeHtmlAttributeChangeListener(HtmlAttributeChangeListener)
1970      */
1971     public void addHtmlAttributeChangeListener(final HtmlAttributeChangeListener listener) {
1972         WebAssert.notNull("listener", listener);
1973         synchronized (lock_) {
1974             if (attributeListeners_ == null) {
1975                 attributeListeners_ = new LinkedHashSet<>();
1976             }
1977             attributeListeners_.add(listener);
1978         }
1979     }
1980 
1981     /**
1982      * Removes an HtmlAttributeChangeListener from the listener list.
1983      * This method should be used to remove HtmlAttributeChangeListener that were registered
1984      * for all attributes of all HtmlElements contained in this page.
1985      *
1986      * @param listener the attribute change listener to be removed
1987      * @see #addHtmlAttributeChangeListener(HtmlAttributeChangeListener)
1988      */
1989     public void removeHtmlAttributeChangeListener(final HtmlAttributeChangeListener listener) {
1990         WebAssert.notNull("listener", listener);
1991         synchronized (lock_) {
1992             if (attributeListeners_ != null) {
1993                 attributeListeners_.remove(listener);
1994             }
1995         }
1996     }
1997 
1998     /**
1999      * Notifies all registered listeners for the given event to add an attribute.
2000      * @param event the event to fire
2001      */
2002     void fireHtmlAttributeAdded(final HtmlAttributeChangeEvent event) {
2003         final List<HtmlAttributeChangeListener> listeners = safeGetAttributeListeners();
2004         if (listeners != null) {
2005             for (final HtmlAttributeChangeListener listener : listeners) {
2006                 listener.attributeAdded(event);
2007             }
2008         }
2009     }
2010 
2011     /**
2012      * Notifies all registered listeners for the given event to replace an attribute.
2013      * @param event the event to fire
2014      */
2015     void fireHtmlAttributeReplaced(final HtmlAttributeChangeEvent event) {
2016         final List<HtmlAttributeChangeListener> listeners = safeGetAttributeListeners();
2017         if (listeners != null) {
2018             for (final HtmlAttributeChangeListener listener : listeners) {
2019                 listener.attributeReplaced(event);
2020             }
2021         }
2022     }
2023 
2024     /**
2025      * Notifies all registered listeners for the given event to remove an attribute.
2026      * @param event the event to fire
2027      */
2028     void fireHtmlAttributeRemoved(final HtmlAttributeChangeEvent event) {
2029         final List<HtmlAttributeChangeListener> listeners = safeGetAttributeListeners();
2030         if (listeners != null) {
2031             for (final HtmlAttributeChangeListener listener : listeners) {
2032                 listener.attributeRemoved(event);
2033             }
2034         }
2035     }
2036 
2037     private List<HtmlAttributeChangeListener> safeGetAttributeListeners() {
2038         synchronized (lock_) {
2039             if (attributeListeners_ != null) {
2040                 return new ArrayList<>(attributeListeners_);
2041             }
2042             return null;
2043         }
2044     }
2045 
2046     /**
2047      * {@inheritDoc}
2048      */
2049     @Override
2050     protected void checkChildHierarchy(final org.w3c.dom.Node newChild) throws DOMException {
2051         if (newChild instanceof Element) {
2052             if (getDocumentElement() != null) {
2053                 throw new DOMException(DOMException.HIERARCHY_REQUEST_ERR,
2054                     "The Document may only have a single child Element.");
2055             }
2056         }
2057         else if (newChild instanceof DocumentType) {
2058             if (getDoctype() != null) {
2059                 throw new DOMException(DOMException.HIERARCHY_REQUEST_ERR,
2060                     "The Document may only have a single child DocumentType.");
2061             }
2062         }
2063         else if (!(newChild instanceof Comment || newChild instanceof ProcessingInstruction)) {
2064             throw new DOMException(DOMException.HIERARCHY_REQUEST_ERR,
2065                 "The Document may not have a child of this type: " + newChild.getNodeType());
2066         }
2067         super.checkChildHierarchy(newChild);
2068     }
2069 
2070     /**
2071      * Returns {@code true} if an HTML parser is operating on this page, adding content to it.
2072      * @return {@code true} if an HTML parser is operating on this page, adding content to it
2073      */
2074     public boolean isBeingParsed() {
2075         return parserCount_ > 0;
2076     }
2077 
2078     /**
2079      * Called by the HTML parser to let the page know that it has started parsing some content for this page.
2080      */
2081     void registerParsingStart() {
2082         parserCount_++;
2083     }
2084 
2085     /**
2086      * Called by the HTML parser to let the page know that it has finished parsing some content for this page.
2087      */
2088     void registerParsingEnd() {
2089         parserCount_--;
2090     }
2091 
2092     /**
2093      * Returns {@code true} if an HTML parser is parsing a non-inline HTML snippet to add content
2094      * to this page. Non-inline content is content that is parsed for the page, but not in the
2095      * same stream as the page itself -- basically anything other than <tt>document.write()</tt>
2096      * or <tt>document.writeln()</tt>: <tt>innerHTML</tt>, <tt>outerHTML</tt>,
2097      * <tt>document.createElement()</tt>, etc.
2098      *
2099      * @return {@code true} if an HTML parser is parsing a non-inline HTML snippet to add content
2100      *         to this page
2101      */
2102     boolean isParsingHtmlSnippet() {
2103         return snippetParserCount_ > 0;
2104     }
2105 
2106     /**
2107      * Called by the HTML parser to let the page know that it has started parsing a non-inline HTML snippet.
2108      */
2109     void registerSnippetParsingStart() {
2110         snippetParserCount_++;
2111     }
2112 
2113     /**
2114      * Called by the HTML parser to let the page know that it has finished parsing a non-inline HTML snippet.
2115      */
2116     void registerSnippetParsingEnd() {
2117         snippetParserCount_--;
2118     }
2119 
2120     /**
2121      * Returns {@code true} if an HTML parser is parsing an inline HTML snippet to add content
2122      * to this page. Inline content is content inserted into the parser stream dynamically
2123      * while the page is being parsed (i.e. <tt>document.write()</tt> or <tt>document.writeln()</tt>).
2124      *
2125      * @return {@code true} if an HTML parser is parsing an inline HTML snippet to add content
2126      *         to this page
2127      */
2128     boolean isParsingInlineHtmlSnippet() {
2129         return inlineSnippetParserCount_ > 0;
2130     }
2131 
2132     /**
2133      * Called by the HTML parser to let the page know that it has started parsing an inline HTML snippet.
2134      */
2135     void registerInlineSnippetParsingStart() {
2136         inlineSnippetParserCount_++;
2137     }
2138 
2139     /**
2140      * Called by the HTML parser to let the page know that it has finished parsing an inline HTML snippet.
2141      */
2142     void registerInlineSnippetParsingEnd() {
2143         inlineSnippetParserCount_--;
2144     }
2145 
2146     /**
2147      * Refreshes the page by sending the same parameters as previously sent to get this page.
2148      * @return the newly loaded page.
2149      * @throws IOException if an IO problem occurs
2150      */
2151     public Page refresh() throws IOException {
2152         return getWebClient().getPage(getWebResponse().getWebRequest());
2153     }
2154 
2155     /**
2156      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2157      * <p>
2158      * Parses the given string as would it belong to the content being parsed
2159      * at the current parsing position
2160      * </p>
2161      * @param string the HTML code to write in place
2162      */
2163     public void writeInParsedStream(final String string) {
2164         builder_.pushInputString(string);
2165     }
2166 
2167     /**
2168      * Sets the builder to allow page to send content from document.write(ln) calls.
2169      * @param htmlUnitDOMBuilder the builder
2170      */
2171     void setBuilder(final HtmlUnitDOMBuilder htmlUnitDOMBuilder) {
2172         builder_ = htmlUnitDOMBuilder;
2173     }
2174 
2175     /**
2176      * Returns the current builder.
2177      * @return the current builder
2178      */
2179     HtmlUnitDOMBuilder getBuilder() {
2180         return builder_;
2181     }
2182 
2183     /**
2184      * <p>Returns all namespaces defined in the root element of this page.</p>
2185      * <p>The default namespace has a key of an empty string.</p>
2186      * @return all namespaces defined in the root element of this page
2187      */
2188     public Map<String, String> getNamespaces() {
2189         final org.w3c.dom.NamedNodeMap attributes = getDocumentElement().getAttributes();
2190         final Map<String, String> namespaces = new HashMap<>();
2191         for (int i = 0; i < attributes.getLength(); i++) {
2192             final Attr attr = (Attr) attributes.item(i);
2193             String name = attr.getName();
2194             if (name.startsWith("xmlns")) {
2195                 int startPos = 5;
2196                 if (name.length() > 5 && name.charAt(5) == ':') {
2197                     startPos = 6;
2198                 }
2199                 name = name.substring(startPos);
2200                 namespaces.put(name, attr.getValue());
2201             }
2202         }
2203         return namespaces;
2204     }
2205 
2206     /**
2207      * {@inheritDoc}
2208      */
2209     @Override
2210     protected void setDocumentType(final DocumentType type) {
2211         super.setDocumentType(type);
2212     }
2213 
2214     /**
2215      * Saves the current page, with all images, to the specified location.
2216      * The default behavior removes all script elements.
2217      *
2218      * @param file file to write this page into
2219      * @throws IOException If an error occurs
2220      */
2221     public void save(final File file) throws IOException {
2222         new XmlSerializer().save(this, file);
2223     }
2224 
2225     /**
2226      * Returns whether the current page mode is in {@code quirks mode} or in {@code standards mode}.
2227      * @return true for {@code quirks mode}, false for {@code standards mode}
2228      */
2229     public boolean isQuirksMode() {
2230         return "BackCompat".equals(((HTMLDocument) getScriptableObject()).getCompatMode());
2231     }
2232 
2233     /**
2234      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2235      * {@inheritDoc}
2236      */
2237     @Override
2238     public boolean isAttachedToPage() {
2239         return true;
2240     }
2241 
2242     /**
2243      * {@inheritDoc}
2244      */
2245     @Override
2246     public boolean isHtmlPage() {
2247         return true;
2248     }
2249 
2250     /**
2251      * The base URL used to resolve relative URLs.
2252      * @return the base URL
2253      */
2254     public URL getBaseURL() {
2255         URL baseUrl;
2256         if (base_ == null) {
2257             baseUrl = getUrl();
2258             final WebWindow window = getEnclosingWindow();
2259             final boolean frame = window != window.getTopWindow();
2260             if (frame) {
2261                 final boolean frameSrcIsNotSet = baseUrl == WebClient.URL_ABOUT_BLANK;
2262                 final boolean frameSrcIsJs = "javascript".equals(baseUrl.getProtocol());
2263                 if (frameSrcIsNotSet || frameSrcIsJs) {
2264                     baseUrl = ((HtmlPage) window.getTopWindow().getEnclosedPage()).getWebResponse()
2265                         .getWebRequest().getUrl();
2266                 }
2267             }
2268             else if (baseUrl_ != null) {
2269                 baseUrl = baseUrl_;
2270             }
2271         }
2272         else {
2273             final String href = base_.getHrefAttribute().trim();
2274             if (StringUtils.isEmpty(href)) {
2275                 baseUrl = getUrl();
2276             }
2277             else {
2278                 final URL url = getUrl();
2279                 try {
2280                     if (href.startsWith("http://") || href.startsWith("https://")) {
2281                         baseUrl = new URL(href);
2282                     }
2283                     else if (href.startsWith("//")) {
2284                         baseUrl = new URL(String.format("%s:%s", url.getProtocol(), href));
2285                     }
2286                     else if (href.startsWith("/")) {
2287                         final int port = Window.getPort(url);
2288                         baseUrl = new URL(String.format("%s://%s:%d%s", url.getProtocol(), url.getHost(), port, href));
2289                     }
2290                     else if (url.toString().endsWith("/")) {
2291                         baseUrl = new URL(String.format("%s%s", url.toString(), href));
2292                     }
2293                     else {
2294                         baseUrl = new URL(UrlUtils.resolveUrl(url, href));
2295                     }
2296                 }
2297                 catch (final MalformedURLException e) {
2298                     notifyIncorrectness("Invalid base url: \"" + href + "\", ignoring it");
2299                     baseUrl = url;
2300                 }
2301             }
2302         }
2303 
2304         return baseUrl;
2305     }
2306 
2307     /**
2308      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2309      *
2310      * Adds an {@link AutoCloseable}, which would be closed during the {@link #cleanUp()}.
2311      * @param autoCloseable the autoclosable
2312      */
2313     public void addAutoCloseable(final AutoCloseable autoCloseable) {
2314         if (autoCloseableList_ == null) {
2315             autoCloseableList_ = new ArrayList<>();
2316         }
2317         autoCloseableList_.add(autoCloseable);
2318     }
2319 
2320     /**
2321      * {@inheritDoc}
2322      */
2323     @Override
2324     public boolean handles(final Event event) {
2325         if (Event.TYPE_BLUR.equals(event.getType()) || Event.TYPE_FOCUS.equals(event.getType())) {
2326             return true;
2327         }
2328         return super.handles(event);
2329     }
2330 
2331     /**
2332      * Sets the {@link ElementFromPointHandler}.
2333      * @param elementFromPointHandler the handler
2334      */
2335     public void setElementFromPointHandler(final ElementFromPointHandler elementFromPointHandler) {
2336         elementFromPointHandler_ = elementFromPointHandler;
2337     }
2338 
2339     /**
2340      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2341      *
2342      * Returns the element for the specified x coordinate and the specified y coordinate.
2343      *
2344      * @param x the x offset, in pixels
2345      * @param y the y offset, in pixels
2346      * @return the element for the specified x coordinate and the specified y coordinate
2347      */
2348     public HtmlElement getElementFromPoint(final int x, final int y) {
2349         if (elementFromPointHandler_ == null) {
2350             LOG.warn("ElementFromPointHandler was not specicifed for " + this);
2351             if (x <= 0 || y <= 0) {
2352                 return null;
2353             }
2354             return getBody();
2355         }
2356         return elementFromPointHandler_.getElementFromPoint(this, x, y);
2357     }
2358 
2359     /**
2360      * Moves the focus to the specified element. This will trigger any relevant JavaScript
2361      * event handlers.
2362      *
2363      * @param newElement the element that will receive the focus, use {@code null} to remove focus from any element
2364      * @return true if the specified element now has the focus
2365      * @see #getFocusedElement()
2366      */
2367     public boolean setFocusedElement(final DomElement newElement) {
2368         return setFocusedElement(newElement, false);
2369     }
2370 
2371     /**
2372      * Moves the focus to the specified element. This will trigger any relevant JavaScript
2373      * event handlers.
2374      *
2375      * @param newElement the element that will receive the focus, use {@code null} to remove focus from any element
2376      * @param windowActivated - whether the enclosing window got focus resulting in specified element getting focus
2377      * @return true if the specified element now has the focus
2378      * @see #getFocusedElement()
2379      */
2380     public boolean setFocusedElement(final DomElement newElement, final boolean windowActivated) {
2381         if (elementWithFocus_ == newElement && !windowActivated) {
2382             // nothing to do
2383             return true;
2384         }
2385 
2386         final DomElement oldFocusedElement = elementWithFocus_;
2387         elementWithFocus_ = null;
2388 
2389         if (!windowActivated) {
2390             if (hasFeature(EVENT_FOCUS_IN_FOCUS_OUT_BLUR)) {
2391                 if (oldFocusedElement != null) {
2392                     oldFocusedElement.fireEvent(Event.TYPE_FOCUS_OUT);
2393                 }
2394 
2395                 if (newElement != null) {
2396                     newElement.fireEvent(Event.TYPE_FOCUS_IN);
2397                 }
2398             }
2399 
2400             if (oldFocusedElement != null) {
2401                 oldFocusedElement.removeFocus();
2402                 oldFocusedElement.fireEvent(Event.TYPE_BLUR);
2403             }
2404         }
2405 
2406         elementWithFocus_ = newElement;
2407 
2408         if (elementWithFocus_ instanceof SelectableTextInput
2409                 && hasFeature(PAGE_SELECTION_RANGE_FROM_SELECTABLE_TEXT_INPUT)) {
2410             final SelectableTextInput sti = (SelectableTextInput) elementWithFocus_;
2411             setSelectionRange(new SimpleRange(sti, sti.getSelectionStart(), sti, sti.getSelectionEnd()));
2412         }
2413 
2414         if (elementWithFocus_ != null) {
2415             elementWithFocus_.focus();
2416             elementWithFocus_.fireEvent(Event.TYPE_FOCUS);
2417         }
2418 
2419         if (hasFeature(EVENT_FOCUS_FOCUS_IN_BLUR_OUT)) {
2420             if (oldFocusedElement != null) {
2421                 oldFocusedElement.fireEvent(Event.TYPE_FOCUS_OUT);
2422             }
2423 
2424             if (newElement != null) {
2425                 newElement.fireEvent(Event.TYPE_FOCUS_IN);
2426             }
2427         }
2428 
2429         // If a page reload happened as a result of the focus change then obviously this
2430         // element will not have the focus because its page has gone away.
2431         return this == getEnclosingWindow().getEnclosedPage();
2432     }
2433 
2434     /**
2435      * Returns the element with the focus or null if no element has the focus.
2436      * @return the element with focus or null
2437      * @see #setFocusedElement(DomElement)
2438      */
2439     public DomElement getFocusedElement() {
2440         return elementWithFocus_;
2441     }
2442 
2443     /**
2444      * <p><span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span></p>
2445      *
2446      * Sets the element with focus.
2447      * @param elementWithFocus the element with focus
2448      */
2449     public void setElementWithFocus(final DomElement elementWithFocus) {
2450         elementWithFocus_ = elementWithFocus;
2451     }
2452 
2453     /**
2454      * <p><span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span></p>
2455      *
2456      * <p>Returns the page's current selection ranges. Note that some browsers, like IE, only allow
2457      * a single selection at a time.</p>
2458      *
2459      * @return the page's current selection ranges
2460      */
2461     public List<Range> getSelectionRanges() {
2462         return selectionRanges_;
2463     }
2464 
2465     /**
2466      * <p><span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span></p>
2467      *
2468      * <p>Makes the specified selection range the *only* selection range on this page.</p>
2469      *
2470      * @param selectionRange the selection range
2471      */
2472     public void setSelectionRange(final Range selectionRange) {
2473         selectionRanges_.clear();
2474         selectionRanges_.add(selectionRange);
2475     }
2476 
2477     /**
2478      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2479      *
2480      * Execute a Function in the given context.
2481      *
2482      * @param function the JavaScript Function to call
2483      * @param thisObject the "this" object to be used during invocation
2484      * @param args the arguments to pass into the call
2485      * @param htmlElementScope the HTML element for which this script is being executed
2486      *        This element will be the context during the JavaScript execution. If null,
2487      *        the context will default to the page.
2488      * @return a ScriptResult which will contain both the current page (which may be different than
2489      *        the previous page and a JavaScript result object.
2490      */
2491     public ScriptResult executeJavaScriptFunction(final Object function, final Object thisObject,
2492             final Object[] args, final DomNode htmlElementScope) {
2493         if (!getWebClient().getOptions().isJavaScriptEnabled()) {
2494             return new ScriptResult(null, this);
2495         }
2496 
2497         return executeJavaScriptFunction((Function) function, (Scriptable) thisObject, args, htmlElementScope);
2498     }
2499 
2500     private ScriptResult executeJavaScriptFunction(final Function function, final Scriptable thisObject,
2501             final Object[] args, final DomNode htmlElementScope) {
2502 
2503         final JavaScriptEngine engine = (JavaScriptEngine) getWebClient().getJavaScriptEngine();
2504         final Object result = engine.callFunction(this, function, thisObject, args, htmlElementScope);
2505 
2506         return new ScriptResult(result, getWebClient().getCurrentWindow().getEnclosedPage());
2507     }
2508 
2509     private void writeObject(final ObjectOutputStream oos) throws IOException {
2510         oos.defaultWriteObject();
2511         oos.writeObject(originalCharset_ == null ? null : originalCharset_.name());
2512     }
2513 
2514     private void readObject(final ObjectInputStream ois) throws ClassNotFoundException, IOException {
2515         ois.defaultReadObject();
2516         final String charsetName = (String) ois.readObject();
2517         if (charsetName != null) {
2518             originalCharset_ = Charset.forName(charsetName);
2519         }
2520     }
2521 }