View Javadoc
1   /*
2    * Copyright (c) 2002-2017 Gargoyle Software Inc.
3    *
4    * Licensed under the Apache License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    * http://www.apache.org/licenses/LICENSE-2.0
8    *
9    * Unless required by applicable law or agreed to in writing, software
10   * distributed under the License is distributed on an "AS IS" BASIS,
11   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12   * See the License for the specific language governing permissions and
13   * limitations under the License.
14   */
15  package com.gargoylesoftware.htmlunit.html;
16  
17  import static com.gargoylesoftware.htmlunit.BrowserVersionFeatures.EVENT_FOCUS_FOCUS_IN_BLUR_OUT;
18  import static com.gargoylesoftware.htmlunit.BrowserVersionFeatures.EVENT_FOCUS_IN_FOCUS_OUT_BLUR;
19  import static com.gargoylesoftware.htmlunit.BrowserVersionFeatures.FOCUS_BODY_ELEMENT_AT_START;
20  import static com.gargoylesoftware.htmlunit.BrowserVersionFeatures.JS_CALL_RESULT_IS_LAST_RETURN_VALUE;
21  import static com.gargoylesoftware.htmlunit.BrowserVersionFeatures.JS_DEFERRED;
22  import static com.gargoylesoftware.htmlunit.BrowserVersionFeatures.JS_IGNORES_UTF8_BOM_SOMETIMES;
23  import static com.gargoylesoftware.htmlunit.BrowserVersionFeatures.PAGE_SELECTION_RANGE_FROM_SELECTABLE_TEXT_INPUT;
24  import static com.gargoylesoftware.htmlunit.BrowserVersionFeatures.URL_MISSING_SLASHES;
25  import static java.nio.charset.StandardCharsets.ISO_8859_1;
26  
27  import java.io.File;
28  import java.io.IOException;
29  import java.io.ObjectInputStream;
30  import java.io.ObjectOutputStream;
31  import java.io.Serializable;
32  import java.net.MalformedURLException;
33  import java.net.URL;
34  import java.nio.charset.Charset;
35  import java.util.ArrayList;
36  import java.util.Arrays;
37  import java.util.Collection;
38  import java.util.Collections;
39  import java.util.Comparator;
40  import java.util.HashMap;
41  import java.util.LinkedHashSet;
42  import java.util.List;
43  import java.util.Locale;
44  import java.util.Map;
45  import java.util.SortedSet;
46  import java.util.TreeSet;
47  
48  import org.apache.commons.lang3.StringUtils;
49  import org.apache.commons.logging.Log;
50  import org.apache.commons.logging.LogFactory;
51  import org.apache.http.HttpStatus;
52  import org.w3c.dom.Attr;
53  import org.w3c.dom.Comment;
54  import org.w3c.dom.DOMConfiguration;
55  import org.w3c.dom.DOMException;
56  import org.w3c.dom.DOMImplementation;
57  import org.w3c.dom.Document;
58  import org.w3c.dom.DocumentType;
59  import org.w3c.dom.Element;
60  import org.w3c.dom.EntityReference;
61  import org.w3c.dom.ProcessingInstruction;
62  import org.w3c.dom.ranges.Range;
63  
64  import com.gargoylesoftware.htmlunit.BrowserVersion;
65  import com.gargoylesoftware.htmlunit.Cache;
66  import com.gargoylesoftware.htmlunit.ElementNotFoundException;
67  import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException;
68  import com.gargoylesoftware.htmlunit.History;
69  import com.gargoylesoftware.htmlunit.OnbeforeunloadHandler;
70  import com.gargoylesoftware.htmlunit.Page;
71  import com.gargoylesoftware.htmlunit.ScriptResult;
72  import com.gargoylesoftware.htmlunit.SgmlPage;
73  import com.gargoylesoftware.htmlunit.TopLevelWindow;
74  import com.gargoylesoftware.htmlunit.WebAssert;
75  import com.gargoylesoftware.htmlunit.WebClient;
76  import com.gargoylesoftware.htmlunit.WebRequest;
77  import com.gargoylesoftware.htmlunit.WebResponse;
78  import com.gargoylesoftware.htmlunit.WebWindow;
79  import com.gargoylesoftware.htmlunit.html.HTMLParser.HtmlUnitDOMBuilder;
80  import com.gargoylesoftware.htmlunit.html.impl.SelectableTextInput;
81  import com.gargoylesoftware.htmlunit.html.impl.SimpleRange;
82  import com.gargoylesoftware.htmlunit.javascript.AbstractJavaScriptEngine;
83  import com.gargoylesoftware.htmlunit.javascript.JavaScriptEngine;
84  import com.gargoylesoftware.htmlunit.javascript.PostponedAction;
85  import com.gargoylesoftware.htmlunit.javascript.SimpleScriptable;
86  import com.gargoylesoftware.htmlunit.javascript.host.Window;
87  import com.gargoylesoftware.htmlunit.javascript.host.dom.Node;
88  import com.gargoylesoftware.htmlunit.javascript.host.event.BeforeUnloadEvent;
89  import com.gargoylesoftware.htmlunit.javascript.host.event.Event;
90  import com.gargoylesoftware.htmlunit.javascript.host.html.HTMLDocument;
91  import com.gargoylesoftware.htmlunit.protocol.javascript.JavaScriptURLConnection;
92  import com.gargoylesoftware.htmlunit.util.EncodingSniffer;
93  import com.gargoylesoftware.htmlunit.util.UrlUtils;
94  
95  import net.sourceforge.htmlunit.corejs.javascript.Context;
96  import net.sourceforge.htmlunit.corejs.javascript.Function;
97  import net.sourceforge.htmlunit.corejs.javascript.Script;
98  import net.sourceforge.htmlunit.corejs.javascript.Scriptable;
99  import net.sourceforge.htmlunit.corejs.javascript.ScriptableObject;
100 import net.sourceforge.htmlunit.corejs.javascript.Undefined;
101 
102 /**
103  * A representation of an HTML page returned from a server.
104  * <p>
105  * This class provides different methods to access the page's content like
106  * {@link #getForms()}, {@link #getAnchors()}, {@link #getElementById(String)}, ... as well as the
107  * very powerful inherited methods {@link #getByXPath(String)} and {@link #getFirstByXPath(String)}
108  * for fine grained user specific access to child nodes.
109  * </p>
110  * <p>
111  * Child elements allowing user interaction provide methods for this purpose like {@link HtmlAnchor#click()},
112  * {@link HtmlInput#type(String)}, {@link HtmlOption#setSelected(boolean)}, ...
113  * </p>
114  * <p>
115  * HtmlPage instances should not be instantiated directly. They will be returned by {@link WebClient#getPage(String)}
116  * when the content type of the server's response is <code>text/html</code> (or one of its variations).<br>
117  * <br>
118  * <b>Example:</b><br>
119  * <br>
120  * <code>
121  * final HtmlPage page = webClient.{@link WebClient#getPage(String) getPage}("http://mywebsite/some/page.html");
122  * </code>
123  * </p>
124  *
125  * @author <a href="mailto:mbowler@GargoyleSoftware.com">Mike Bowler</a>
126  * @author Alex Nikiforoff
127  * @author Noboru Sinohara
128  * @author David K. Taylor
129  * @author Andreas Hangler
130  * @author <a href="mailto:cse@dynabean.de">Christian Sell</a>
131  * @author Chris Erskine
132  * @author Marc Guillemot
133  * @author Ahmed Ashour
134  * @author Daniel Gredler
135  * @author Dmitri Zoubkov
136  * @author Sudhan Moghe
137  * @author Ethan Glasser-Camp
138  * @author <a href="mailto:tom.anderson@univ.oxon.org">Tom Anderson</a>
139  * @author Ronald Brill
140  * @author Frank Danek
141  * @author Joerg Werner
142  */
143 public class HtmlPage extends SgmlPage {
144 
145     private static final Log LOG = LogFactory.getLog(HtmlPage.class);
146 
147     private static final Comparator<DomElement> documentPositionComparator = new DocumentPositionComparator();
148 
149     private HtmlUnitDOMBuilder builder_;
150     private transient Charset originalCharset_;
151 
152     private Map<String, SortedSet<DomElement>> idMap_
153             = Collections.synchronizedMap(new HashMap<String, SortedSet<DomElement>>());
154     private Map<String, SortedSet<DomElement>> nameMap_
155             = Collections.synchronizedMap(new HashMap<String, SortedSet<DomElement>>());
156 
157     private SortedSet<BaseFrameElement> frameElements_ = new TreeSet<>(documentPositionComparator);
158     private int parserCount_;
159     private int snippetParserCount_;
160     private int inlineSnippetParserCount_;
161     private Collection<HtmlAttributeChangeListener> attributeListeners_;
162     private final Object lock_ = new String(); // used for synchronization
163     private List<PostponedAction> afterLoadActions_ = Collections.synchronizedList(new ArrayList<PostponedAction>());
164     private boolean cleaning_;
165     private HtmlBase base_;
166     private URL baseUrl_;
167     private List<AutoCloseable> autoCloseableList_;
168     private ElementFromPointHandler elementFromPointHandler_;
169     private DomElement elementWithFocus_;
170     private List<Range> selectionRanges_ = new ArrayList<>(3);
171 
172     private static final List<String> TABBABLE_TAGS = Arrays.asList(HtmlAnchor.TAG_NAME, HtmlArea.TAG_NAME,
173             HtmlButton.TAG_NAME, HtmlInput.TAG_NAME, HtmlObject.TAG_NAME, HtmlSelect.TAG_NAME, HtmlTextArea.TAG_NAME);
174     private static final List<String> ACCEPTABLE_TAG_NAMES = Arrays.asList(HtmlAnchor.TAG_NAME, HtmlArea.TAG_NAME,
175             HtmlButton.TAG_NAME, HtmlInput.TAG_NAME, HtmlLabel.TAG_NAME, HtmlLegend.TAG_NAME, HtmlTextArea.TAG_NAME);
176 
177     static class DocumentPositionComparator implements Comparator<DomElement>, Serializable {
178         @Override
179         public int compare(final DomElement elt1, final DomElement elt2) {
180             final short relation = elt1.compareDocumentPosition(elt2);
181             if (relation == 0) {
182                 return 0; // same node
183             }
184             if ((relation & DOCUMENT_POSITION_CONTAINS) != 0 || (relation & DOCUMENT_POSITION_PRECEDING) != 0) {
185                 return 1;
186             }
187 
188             return -1;
189         }
190     }
191 
192     /**
193      * Creates an instance of HtmlPage.
194      * An HtmlPage instance is normally retrieved with {@link WebClient#getPage(String)}.
195      *
196      * @param webResponse the web response that was used to create this page
197      * @param webWindow the window that this page is being loaded into
198      */
199     public HtmlPage(final WebResponse webResponse, final WebWindow webWindow) {
200         super(webResponse, webWindow);
201     }
202 
203     /**
204      * {@inheritDoc}
205      */
206     @Override
207     public HtmlPage getPage() {
208         return this;
209     }
210 
211     /**
212      * {@inheritDoc}
213      */
214     @Override
215     public boolean hasCaseSensitiveTagNames() {
216         return false;
217     }
218 
219     /**
220      * Initialize this page.
221      * @throws IOException if an IO problem occurs
222      * @throws FailingHttpStatusCodeException if the server returns a failing status code AND the property
223      * {@link com.gargoylesoftware.htmlunit.WebClientOptions#setThrowExceptionOnFailingStatusCode(boolean)} is set
224      * to true.
225      */
226     @Override
227     public void initialize() throws IOException, FailingHttpStatusCodeException {
228         final WebWindow enclosingWindow = getEnclosingWindow();
229         final boolean isAboutBlank = getUrl() == WebClient.URL_ABOUT_BLANK;
230         if (isAboutBlank) {
231             // a frame contains first a faked "about:blank" before its real content specified by src gets loaded
232             if (enclosingWindow instanceof FrameWindow
233                     && !((FrameWindow) enclosingWindow).getFrameElement().isContentLoaded()) {
234                 return;
235             }
236 
237             // save the URL that should be used to resolve relative URLs in this page
238             if (enclosingWindow instanceof TopLevelWindow) {
239                 final TopLevelWindow topWindow = (TopLevelWindow) enclosingWindow;
240                 final WebWindow openerWindow = topWindow.getOpener();
241                 if (openerWindow != null && openerWindow.getEnclosedPage() != null) {
242                     baseUrl_ = openerWindow.getEnclosedPage().getWebResponse().getWebRequest().getUrl();
243                 }
244             }
245         }
246         loadFrames();
247 
248         // don't set the ready state if we really load the blank page into the window
249         // see Node.initInlineFrameIfNeeded()
250         if (!isAboutBlank) {
251             if (hasFeature(FOCUS_BODY_ELEMENT_AT_START)) {
252                 setElementWithFocus(getBody());
253             }
254             setReadyState(READY_STATE_COMPLETE);
255             getDocumentElement().setReadyState(READY_STATE_COMPLETE);
256         }
257 
258         executeEventHandlersIfNeeded(Event.TYPE_DOM_DOCUMENT_LOADED);
259         executeDeferredScriptsIfNeeded();
260         setReadyStateOnDeferredScriptsIfNeeded();
261 
262         // frame initialization has a different order
263         boolean isFrameWindow = enclosingWindow instanceof FrameWindow;
264         boolean isFirstPageInFrameWindow = false;
265         if (isFrameWindow) {
266             isFrameWindow = ((FrameWindow) enclosingWindow).getFrameElement() instanceof HtmlFrame;
267 
268             final History hist = enclosingWindow.getHistory();
269             if (hist.getLength() > 0 && WebClient.URL_ABOUT_BLANK == hist.getUrl(0)) {
270                 isFirstPageInFrameWindow = hist.getLength() <= 2;
271             }
272             else {
273                 isFirstPageInFrameWindow = enclosingWindow.getHistory().getLength() < 2;
274             }
275         }
276 
277         if (isFrameWindow && !isFirstPageInFrameWindow) {
278             executeEventHandlersIfNeeded(Event.TYPE_LOAD);
279         }
280 
281         for (final FrameWindow frameWindow : getFrames()) {
282             if (frameWindow.getFrameElement() instanceof HtmlFrame) {
283                 final Page page = frameWindow.getEnclosedPage();
284                 if (page != null && page.isHtmlPage()) {
285                     ((HtmlPage) page).executeEventHandlersIfNeeded(Event.TYPE_LOAD);
286                 }
287             }
288         }
289 
290         if (!isFrameWindow) {
291             executeEventHandlersIfNeeded(Event.TYPE_LOAD);
292         }
293 
294         try {
295             while (!afterLoadActions_.isEmpty()) {
296                 final PostponedAction action = afterLoadActions_.remove(0);
297                 action.execute();
298             }
299         }
300         catch (final IOException e) {
301             throw e;
302         }
303         catch (final Exception e) {
304             throw new RuntimeException(e);
305         }
306         executeRefreshIfNeeded();
307     }
308 
309     /**
310      * Adds an action that should be executed once the page has been loaded.
311      * @param action the action
312      */
313     void addAfterLoadAction(final PostponedAction action) {
314         afterLoadActions_.add(action);
315     }
316 
317     /**
318      * Clean up this page.
319      */
320     @Override
321     public void cleanUp() {
322         //To avoid endless recursion caused by window.close() in onUnload
323         if (cleaning_) {
324             return;
325         }
326         cleaning_ = true;
327         super.cleanUp();
328         executeEventHandlersIfNeeded(Event.TYPE_UNLOAD);
329         deregisterFramesIfNeeded();
330         cleaning_ = false;
331         if (autoCloseableList_ != null) {
332             for (final AutoCloseable closeable : new ArrayList<>(autoCloseableList_)) {
333                 try {
334                     closeable.close();
335                 }
336                 catch (final Exception e) {
337                     throw new RuntimeException(e);
338                 }
339             }
340         }
341     }
342 
343     /**
344      * {@inheritDoc}
345      */
346     @Override
347     public HtmlElement getDocumentElement() {
348         return (HtmlElement) super.getDocumentElement();
349     }
350 
351     /**
352      * Returns the <tt>body</tt> element (or <tt>frameset</tt> element), or {@code null} if it does not yet exist.
353      * @return the <tt>body</tt> element (or <tt>frameset</tt> element), or {@code null} if it does not yet exist
354      */
355     public HtmlElement getBody() {
356         final DomElement doc = getDocumentElement();
357         if (doc != null) {
358             for (final DomNode node : doc.getChildren()) {
359                 if (node instanceof HtmlBody || node instanceof HtmlFrameSet) {
360                     return (HtmlElement) node;
361                 }
362             }
363         }
364         return null;
365     }
366 
367     /**
368      * Returns the head element.
369      * @return the head element
370      */
371     public HtmlElement getHead() {
372         final DomElement doc = getDocumentElement();
373         if (doc != null) {
374             for (final DomNode node : doc.getChildren()) {
375                 if (node instanceof HtmlHead) {
376                     return (HtmlElement) node;
377                 }
378             }
379         }
380         return null;
381     }
382 
383     /**
384      * {@inheritDoc}
385      */
386     @Override
387     public Document getOwnerDocument() {
388         return null;
389     }
390 
391     /**
392      * {@inheritDoc}
393      * Not yet implemented.
394      */
395     @Override
396     public org.w3c.dom.Node importNode(final org.w3c.dom.Node importedNode, final boolean deep) {
397         throw new UnsupportedOperationException("HtmlPage.importNode is not yet implemented.");
398     }
399 
400     /**
401      * {@inheritDoc}
402      * Not yet implemented.
403      */
404     @Override
405     public String getInputEncoding() {
406         throw new UnsupportedOperationException("HtmlPage.getInputEncoding is not yet implemented.");
407     }
408 
409     /**
410      * {@inheritDoc}
411      */
412     @Override
413     public String getXmlEncoding() {
414         return null;
415     }
416 
417     /**
418      * {@inheritDoc}
419      */
420     @Override
421     public boolean getXmlStandalone() {
422         return false;
423     }
424 
425     /**
426      * {@inheritDoc}
427      * Not yet implemented.
428      */
429     @Override
430     public void setXmlStandalone(final boolean xmlStandalone) throws DOMException {
431         throw new UnsupportedOperationException("HtmlPage.setXmlStandalone is not yet implemented.");
432     }
433 
434     /**
435      * {@inheritDoc}
436      */
437     @Override
438     public String getXmlVersion() {
439         return null;
440     }
441 
442     /**
443      * {@inheritDoc}
444      * Not yet implemented.
445      */
446     @Override
447     public void setXmlVersion(final String xmlVersion) throws DOMException {
448         throw new UnsupportedOperationException("HtmlPage.setXmlVersion is not yet implemented.");
449     }
450 
451     /**
452      * {@inheritDoc}
453      * Not yet implemented.
454      */
455     @Override
456     public boolean getStrictErrorChecking() {
457         throw new UnsupportedOperationException("HtmlPage.getStrictErrorChecking is not yet implemented.");
458     }
459 
460     /**
461      * {@inheritDoc}
462      * Not yet implemented.
463      */
464     @Override
465     public void setStrictErrorChecking(final boolean strictErrorChecking) {
466         throw new UnsupportedOperationException("HtmlPage.setStrictErrorChecking is not yet implemented.");
467     }
468 
469     /**
470      * {@inheritDoc}
471      * Not yet implemented.
472      */
473     @Override
474     public String getDocumentURI() {
475         throw new UnsupportedOperationException("HtmlPage.getDocumentURI is not yet implemented.");
476     }
477 
478     /**
479      * {@inheritDoc}
480      * Not yet implemented.
481      */
482     @Override
483     public void setDocumentURI(final String documentURI) {
484         throw new UnsupportedOperationException("HtmlPage.setDocumentURI is not yet implemented.");
485     }
486 
487     /**
488      * {@inheritDoc}
489      * Not yet implemented.
490      */
491     @Override
492     public org.w3c.dom.Node adoptNode(final org.w3c.dom.Node source) throws DOMException {
493         throw new UnsupportedOperationException("HtmlPage.adoptNode is not yet implemented.");
494     }
495 
496     /**
497      * {@inheritDoc}
498      * Not yet implemented.
499      */
500     @Override
501     public DOMConfiguration getDomConfig() {
502         throw new UnsupportedOperationException("HtmlPage.getDomConfig is not yet implemented.");
503     }
504 
505     /**
506      * {@inheritDoc}
507      * Not yet implemented.
508      */
509     @Override
510     public org.w3c.dom.Node renameNode(final org.w3c.dom.Node newNode, final String namespaceURI,
511         final String qualifiedName) throws DOMException {
512         throw new UnsupportedOperationException("HtmlPage.renameNode is not yet implemented.");
513     }
514 
515     /**
516      * {@inheritDoc}
517      */
518     @Override
519     public Charset getCharset() {
520         if (originalCharset_ == null) {
521             originalCharset_ = getWebResponse().getContentCharset();
522         }
523         return originalCharset_;
524     }
525 
526     /**
527      * {@inheritDoc}
528      */
529     @Override
530     public String getContentType() {
531         return getWebResponse().getContentType();
532     }
533 
534     /**
535      * {@inheritDoc}
536      * Not yet implemented.
537      */
538     @Override
539     public DOMImplementation getImplementation() {
540         throw new UnsupportedOperationException("HtmlPage.getImplementation is not yet implemented.");
541     }
542 
543     /**
544      * {@inheritDoc}
545      * @param tagName the tag name, preferably in lowercase
546      */
547     @Override
548     public DomElement createElement(String tagName) {
549         if (tagName.indexOf(':') == -1) {
550             tagName = tagName.toLowerCase(Locale.ROOT);
551         }
552         return HTMLParser.getFactory(tagName).createElementNS(this, null, tagName, null, true);
553     }
554 
555     /**
556      * {@inheritDoc}
557      */
558     @Override
559     public DomElement createElementNS(final String namespaceURI, final String qualifiedName) {
560         return HTMLParser.getElementFactory(this, namespaceURI, qualifiedName, false, true)
561             .createElementNS(this, namespaceURI, qualifiedName, null, true);
562     }
563 
564     /**
565      * {@inheritDoc}
566      * Not yet implemented.
567      */
568     @Override
569     public Attr createAttributeNS(final String namespaceURI, final String qualifiedName) {
570         throw new UnsupportedOperationException("HtmlPage.createAttributeNS is not yet implemented.");
571     }
572 
573     /**
574      * {@inheritDoc}
575      * Not yet implemented.
576      */
577     @Override
578     public EntityReference createEntityReference(final String id) {
579         throw new UnsupportedOperationException("HtmlPage.createEntityReference is not yet implemented.");
580     }
581 
582     /**
583      * {@inheritDoc}
584      * Not yet implemented.
585      */
586     @Override
587     public ProcessingInstruction createProcessingInstruction(final String namespaceURI, final String qualifiedName) {
588         throw new UnsupportedOperationException("HtmlPage.createProcessingInstruction is not yet implemented.");
589     }
590 
591     /**
592      * {@inheritDoc}
593      */
594     @Override
595     public DomElement getElementById(final String elementId) {
596         final SortedSet<DomElement> elements = idMap_.get(elementId);
597         if (elements != null) {
598             return elements.first();
599         }
600         return null;
601     }
602 
603     /**
604      * Returns the {@link HtmlAnchor} with the specified name.
605      *
606      * @param name the name to search by
607      * @return the {@link HtmlAnchor} with the specified name
608      * @throws ElementNotFoundException if the anchor could not be found
609      */
610     public HtmlAnchor getAnchorByName(final String name) throws ElementNotFoundException {
611         return getDocumentElement().getOneHtmlElementByAttribute("a", "name", name);
612     }
613 
614     /**
615      * Returns the {@link HtmlAnchor} with the specified href.
616      *
617      * @param href the string to search by
618      * @return the HtmlAnchor
619      * @throws ElementNotFoundException if the anchor could not be found
620      */
621     public HtmlAnchor getAnchorByHref(final String href) throws ElementNotFoundException {
622         return getDocumentElement().getOneHtmlElementByAttribute("a", "href", href);
623     }
624 
625     /**
626      * Returns a list of all anchors contained in this page.
627      * @return the list of {@link HtmlAnchor} in this page
628      */
629     public List<HtmlAnchor> getAnchors() {
630         return getDocumentElement().getElementsByTagNameImpl("a");
631     }
632 
633     /**
634      * Returns the first anchor with the specified text.
635      * @param text the text to search for
636      * @return the first anchor that was found
637      * @throws ElementNotFoundException if no anchors are found with the specified text
638      */
639     public HtmlAnchor getAnchorByText(final String text) throws ElementNotFoundException {
640         WebAssert.notNull("text", text);
641 
642         for (final HtmlAnchor anchor : getAnchors()) {
643             if (text.equals(anchor.asText())) {
644                 return anchor;
645             }
646         }
647         throw new ElementNotFoundException("a", "<text>", text);
648     }
649 
650     /**
651      * Returns the first form that matches the specified name.
652      * @param name the name to search for
653      * @return the first form
654      * @exception ElementNotFoundException If no forms match the specified result.
655      */
656     public HtmlForm getFormByName(final String name) throws ElementNotFoundException {
657         final List<HtmlForm> forms = getDocumentElement().getElementsByAttribute("form", "name", name);
658         if (forms.isEmpty()) {
659             throw new ElementNotFoundException("form", "name", name);
660         }
661         return forms.get(0);
662     }
663 
664     /**
665      * Returns a list of all the forms in this page.
666      * @return all the forms in this page
667      */
668     public List<HtmlForm> getForms() {
669         return getDocumentElement().getElementsByTagNameImpl("form");
670     }
671 
672     /**
673      * Given a relative URL (ie <tt>/foo</tt>), returns a fully-qualified URL based on
674      * the URL that was used to load this page.
675      *
676      * @param relativeUrl the relative URL
677      * @return the fully-qualified URL for the specified relative URL
678      * @exception MalformedURLException if an error occurred when creating a URL object
679      */
680     public URL getFullyQualifiedUrl(String relativeUrl) throws MalformedURLException {
681         final URL baseUrl = getBaseURL();
682 
683         // to handle http: and http:/ in FF (Bug #474)
684         if (hasFeature(URL_MISSING_SLASHES)) {
685             boolean incorrectnessNotified = false;
686             while (relativeUrl.startsWith("http:") && !relativeUrl.startsWith("http://")) {
687                 if (!incorrectnessNotified) {
688                     notifyIncorrectness("Incorrect URL \"" + relativeUrl + "\" has been corrected");
689                     incorrectnessNotified = true;
690                 }
691                 relativeUrl = "http:/" + relativeUrl.substring(5);
692             }
693         }
694 
695         return WebClient.expandUrl(baseUrl, relativeUrl);
696     }
697 
698     /**
699      * Given a target attribute value, resolve the target using a base target for the page.
700      *
701      * @param elementTarget the target specified as an attribute of the element
702      * @return the resolved target to use for the element
703      */
704     public String getResolvedTarget(final String elementTarget) {
705         final String resolvedTarget;
706         if (base_ == null) {
707             resolvedTarget = elementTarget;
708         }
709         else if (elementTarget != null && !elementTarget.isEmpty()) {
710             resolvedTarget = elementTarget;
711         }
712         else {
713             resolvedTarget = base_.getTargetAttribute();
714         }
715         return resolvedTarget;
716     }
717 
718     /**
719      * Returns a list of ids (strings) that correspond to the tabbable elements
720      * in this page. Return them in the same order specified in {@link #getTabbableElements}
721      *
722      * @return the list of id's
723      */
724     public List<String> getTabbableElementIds() {
725         final List<String> list = new ArrayList<>();
726 
727         for (final HtmlElement element : getTabbableElements()) {
728             list.add(element.getId());
729         }
730 
731         return Collections.unmodifiableList(list);
732     }
733 
734     /**
735      * Returns a list of all elements that are tabbable in the order that will
736      * be used for tabbing.<p>
737      *
738      * The rules for determining tab order are as follows:
739      * <ol>
740      *   <li>Those elements that support the tabindex attribute and assign a
741      *   positive value to it are navigated first. Navigation proceeds from the
742      *   element with the lowest tabindex value to the element with the highest
743      *   value. Values need not be sequential nor must they begin with any
744      *   particular value. Elements that have identical tabindex values should
745      *   be navigated in the order they appear in the character stream.
746      *   <li>Those elements that do not support the tabindex attribute or
747      *   support it and assign it a value of "0" are navigated next. These
748      *   elements are navigated in the order they appear in the character
749      *   stream.
750      *   <li>Elements that are disabled do not participate in the tabbing
751      *   order.
752      * </ol>
753      * Additionally, the value of tabindex must be within 0 and 32767. Any
754      * values outside this range will be ignored.<p>
755      *
756      * The following elements support the <tt>tabindex</tt> attribute: A, AREA, BUTTON,
757      * INPUT, OBJECT, SELECT, and TEXTAREA.<p>
758      *
759      * @return all the tabbable elements in proper tab order
760      */
761     public List<HtmlElement> getTabbableElements() {
762         final List<HtmlElement> tabbableElements = new ArrayList<>();
763         for (final HtmlElement element : getHtmlElementDescendants()) {
764             final String tagName = element.getTagName();
765             if (TABBABLE_TAGS.contains(tagName)) {
766                 final boolean disabled = element.hasAttribute("disabled");
767                 if (!disabled && element.getTabIndex() != HtmlElement.TAB_INDEX_OUT_OF_BOUNDS) {
768                     tabbableElements.add(element);
769                 }
770             }
771         }
772         Collections.sort(tabbableElements, createTabOrderComparator());
773         return Collections.unmodifiableList(tabbableElements);
774     }
775 
776     private static Comparator<HtmlElement> createTabOrderComparator() {
777         return new Comparator<HtmlElement>() {
778             @Override
779             public int compare(final HtmlElement element1, final HtmlElement element2) {
780                 final Short i1 = element1.getTabIndex();
781                 final Short i2 = element2.getTabIndex();
782 
783                 final short index1;
784                 if (i1 != null) {
785                     index1 = i1.shortValue();
786                 }
787                 else {
788                     index1 = -1;
789                 }
790 
791                 final short index2;
792                 if (i2 != null) {
793                     index2 = i2.shortValue();
794                 }
795                 else {
796                     index2 = -1;
797                 }
798 
799                 final int result;
800                 if (index1 > 0 && index2 > 0) {
801                     result = index1 - index2;
802                 }
803                 else if (index1 > 0) {
804                     result = -1;
805                 }
806                 else if (index2 > 0) {
807                     result = +1;
808                 }
809                 else if (index1 == index2) {
810                     result = 0;
811                 }
812                 else {
813                     result = index2 - index1;
814                 }
815 
816                 return result;
817             }
818         };
819     }
820 
821     /**
822      * Returns the HTML element that is assigned to the specified access key. An
823      * access key (aka mnemonic key) is used for keyboard navigation of the
824      * page.<p>
825      *
826      * Only the following HTML elements may have <tt>accesskey</tt>s defined: A, AREA,
827      * BUTTON, INPUT, LABEL, LEGEND, and TEXTAREA.
828      *
829      * @param accessKey the key to look for
830      * @return the HTML element that is assigned to the specified key or null
831      *      if no elements can be found that match the specified key.
832      */
833     public HtmlElement getHtmlElementByAccessKey(final char accessKey) {
834         final List<HtmlElement> elements = getHtmlElementsByAccessKey(accessKey);
835         if (elements.isEmpty()) {
836             return null;
837         }
838         return elements.get(0);
839     }
840 
841     /**
842      * Returns all the HTML elements that are assigned to the specified access key. An
843      * access key (aka mnemonic key) is used for keyboard navigation of the
844      * page.<p>
845      *
846      * The HTML specification seems to indicate that one accesskey cannot be used
847      * for multiple elements however Internet Explorer does seem to support this.
848      * It's worth noting that Firefox does not support multiple elements with one
849      * access key so you are making your HTML browser specific if you rely on this
850      * feature.<p>
851      *
852      * Only the following HTML elements may have <tt>accesskey</tt>s defined: A, AREA,
853      * BUTTON, INPUT, LABEL, LEGEND, and TEXTAREA.
854      *
855      * @param accessKey the key to look for
856      * @return the elements that are assigned to the specified accesskey
857      */
858     public List<HtmlElement> getHtmlElementsByAccessKey(final char accessKey) {
859         final List<HtmlElement> elements = new ArrayList<>();
860 
861         final String searchString = Character.toString(accessKey).toLowerCase(Locale.ROOT);
862         for (final HtmlElement element : getHtmlElementDescendants()) {
863             if (ACCEPTABLE_TAG_NAMES.contains(element.getTagName())) {
864                 final String accessKeyAttribute = element.getAttribute("accesskey");
865                 if (searchString.equalsIgnoreCase(accessKeyAttribute)) {
866                     elements.add(element);
867                 }
868             }
869         }
870 
871         return elements;
872     }
873 
874     /**
875      * <p>Executes the specified JavaScript code within the page. The usage would be similar to what can
876      * be achieved to execute JavaScript in the current page by entering "javascript:...some JS code..."
877      * in the URL field of a native browser.</p>
878      * <p><b>Note:</b> the provided code won't be executed if JavaScript has been disabled on the WebClient
879      * (see {@link com.gargoylesoftware.htmlunit.WebClientOptions#isJavaScriptEnabled()}.</p>
880      * @param sourceCode the JavaScript code to execute
881      * @return a ScriptResult which will contain both the current page (which may be different than
882      * the previous page) and a JavaScript result object
883      */
884     public ScriptResult executeJavaScript(final String sourceCode) {
885         return executeJavaScript(sourceCode, "injected script", 1);
886     }
887 
888     /**
889      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
890      * <p>
891      * Execute the specified JavaScript if a JavaScript engine was successfully
892      * instantiated. If this JavaScript causes the current page to be reloaded
893      * (through location="" or form.submit()) then return the new page. Otherwise
894      * return the current page.
895      * </p>
896      * <p><b>Please note:</b> Although this method is public, it is not intended for
897      * general execution of JavaScript. Users of HtmlUnit should interact with the pages
898      * as a user would by clicking on buttons or links and having the JavaScript event
899      * handlers execute as needed..
900      * </p>
901      *
902      * @param sourceCode the JavaScript code to execute
903      * @param sourceName the name for this chunk of code (will be displayed in error messages)
904      * @param startLine the line at which the script source starts
905      * @return a ScriptResult which will contain both the current page (which may be different than
906      * the previous page and a JavaScript result object.
907      */
908     public ScriptResult executeJavaScript(String sourceCode, final String sourceName, final int startLine) {
909         if (!getWebClient().getOptions().isJavaScriptEnabled()) {
910             return new ScriptResult(null, this);
911         }
912 
913         if (StringUtils.startsWithIgnoreCase(sourceCode, JavaScriptURLConnection.JAVASCRIPT_PREFIX)) {
914             sourceCode = sourceCode.substring(JavaScriptURLConnection.JAVASCRIPT_PREFIX.length()).trim();
915             if (sourceCode.startsWith("return ")) {
916                 sourceCode = sourceCode.substring("return ".length());
917             }
918         }
919 
920         final Object result = getWebClient().getJavaScriptEngine().execute(this, sourceCode, sourceName, startLine);
921         return new ScriptResult(result, getWebClient().getCurrentWindow().getEnclosedPage());
922     }
923 
924     /** Various possible external JavaScript file loading results. */
925     enum JavaScriptLoadResult {
926         /** The load was aborted and nothing was done. */
927         NOOP,
928         /** The external JavaScript file was downloaded and compiled successfully. */
929         SUCCESS,
930         /** The external JavaScript file was not downloaded successfully. */
931         DOWNLOAD_ERROR,
932         /** The external JavaScript file was downloaded but was not compiled successfully. */
933         COMPILATION_ERROR
934     }
935 
936     /**
937      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
938      *
939      * @param srcAttribute the source attribute from the script tag
940      * @param scriptCharset the charset from the script tag
941      * @return the result of loading the specified external JavaScript file
942      * @throws FailingHttpStatusCodeException if the request's status code indicates a request
943      *         failure and the {@link WebClient} was configured to throw exceptions on failing
944      *         HTTP status codes
945      */
946     JavaScriptLoadResult loadExternalJavaScriptFile(final String srcAttribute, final Charset scriptCharset)
947         throws FailingHttpStatusCodeException {
948 
949         final WebClient client = getWebClient();
950         if (StringUtils.isBlank(srcAttribute) || !client.getOptions().isJavaScriptEnabled()) {
951             return JavaScriptLoadResult.NOOP;
952         }
953 
954         final URL scriptURL;
955         try {
956             scriptURL = getFullyQualifiedUrl(srcAttribute);
957             final String protocol = scriptURL.getProtocol();
958             if ("javascript".equals(protocol)) {
959                 LOG.info("Ignoring script src [" + srcAttribute + "]");
960                 return JavaScriptLoadResult.NOOP;
961             }
962             if (!"http".equals(protocol) && !"https".equals(protocol)
963                     && !"data".equals(protocol) && !"file".equals(protocol)) {
964                 client.getJavaScriptErrorListener().malformedScriptURL(this, srcAttribute,
965                         new MalformedURLException("unknown protocol: '" + protocol + "'"));
966                 return JavaScriptLoadResult.NOOP;
967             }
968         }
969         catch (final MalformedURLException e) {
970             client.getJavaScriptErrorListener().malformedScriptURL(this, srcAttribute, e);
971             return JavaScriptLoadResult.NOOP;
972         }
973 
974         final Object script;
975         try {
976             script = loadJavaScriptFromUrl(scriptURL, scriptCharset);
977         }
978         catch (final IOException e) {
979             client.getJavaScriptErrorListener().loadScriptError(this, scriptURL, e);
980             return JavaScriptLoadResult.DOWNLOAD_ERROR;
981         }
982         catch (final FailingHttpStatusCodeException e) {
983             client.getJavaScriptErrorListener().loadScriptError(this, scriptURL, e);
984             throw e;
985         }
986 
987         if (script == null) {
988             return JavaScriptLoadResult.COMPILATION_ERROR;
989         }
990 
991         @SuppressWarnings("unchecked")
992         final AbstractJavaScriptEngine<Object> engine = (AbstractJavaScriptEngine<Object>) client.getJavaScriptEngine();
993         engine.execute(this, script);
994         return JavaScriptLoadResult.SUCCESS;
995     }
996 
997     /**
998      * Loads JavaScript from the specified URL. This method may return {@code null} if
999      * there is a problem loading the code from the specified URL.
1000      *
1001      * @param url the URL of the script
1002      * @param scriptCharset the charset from the script tag
1003      * @return the content of the file, or {@code null} if we ran into a compile error
1004      * @throws IOException if there is a problem downloading the JavaScript file
1005      * @throws FailingHttpStatusCodeException if the request's status code indicates a request
1006      *         failure and the {@link WebClient} was configured to throw exceptions on failing
1007      *         HTTP status codes
1008      */
1009     private Object loadJavaScriptFromUrl(final URL url, final Charset scriptCharset) throws IOException,
1010         FailingHttpStatusCodeException {
1011 
1012         final WebRequest referringRequest = getWebResponse().getWebRequest();
1013 
1014         final WebClient client = getWebClient();
1015         final BrowserVersion browser = client.getBrowserVersion();
1016         final WebRequest request = new WebRequest(url, browser.getScriptAcceptHeader());
1017         request.setAdditionalHeaders(new HashMap<>(referringRequest.getAdditionalHeaders()));
1018         request.setAdditionalHeader("Referer", referringRequest.getUrl().toString());
1019         request.setAdditionalHeader("Accept", client.getBrowserVersion().getScriptAcceptHeader());
1020 
1021         // our cache is a bit strange;
1022         // loadWebResponse check the cache for the web response
1023         // AND also fixes the request url for the following cache lookups
1024         final WebResponse response = client.loadWebResponse(request);
1025 
1026         // now we can look into the cache with the fixed request for
1027         // a cached script
1028         final Cache cache = client.getCache();
1029         final Object cachedScript = cache.getCachedObject(request);
1030         if (cachedScript instanceof Script) {
1031             return cachedScript;
1032         }
1033 
1034         client.printContentIfNecessary(response);
1035         client.throwFailingHttpStatusCodeExceptionIfNecessary(response);
1036 
1037         final int statusCode = response.getStatusCode();
1038         final boolean successful = statusCode >= HttpStatus.SC_OK && statusCode < HttpStatus.SC_MULTIPLE_CHOICES;
1039         final boolean noContent = statusCode == HttpStatus.SC_NO_CONTENT;
1040         if (!successful || noContent) {
1041             throw new IOException("Unable to download JavaScript from '" + url + "' (status " + statusCode + ").");
1042         }
1043 
1044         //http://www.ietf.org/rfc/rfc4329.txt
1045         final String contentType = response.getContentType();
1046         if (!"application/javascript".equalsIgnoreCase(contentType)
1047             && !"application/ecmascript".equalsIgnoreCase(contentType)) {
1048             // warn about obsolete or not supported content types
1049             if ("text/javascript".equals(contentType)
1050                     || "text/ecmascript".equals(contentType)
1051                     || "application/x-javascript".equalsIgnoreCase(contentType)) {
1052                 getWebClient().getIncorrectnessListener().notify(
1053                     "Obsolete content type encountered: '" + contentType + "'.", this);
1054             }
1055             else {
1056                 getWebClient().getIncorrectnessListener().notify(
1057                         "Expected content type of 'application/javascript' or 'application/ecmascript' for "
1058                         + "remotely loaded JavaScript element at '" + url + "', "
1059                         + "but got '" + contentType + "'.", this);
1060             }
1061         }
1062 
1063         Charset scriptEncoding = Charset.forName("windows-1252");
1064         boolean ignoreBom = false;
1065         final Charset contentCharset = EncodingSniffer.sniffEncodingFromHttpHeaders(response.getResponseHeaders());
1066         if (contentCharset == null) {
1067             // use info from script tag or fall back to utf-8
1068             if (scriptCharset != null && ISO_8859_1 != scriptCharset) {
1069                 ignoreBom = true;
1070                 scriptEncoding = scriptCharset;
1071             }
1072             else {
1073                 ignoreBom = ISO_8859_1 != scriptCharset;
1074             }
1075         }
1076         else if (ISO_8859_1 != contentCharset) {
1077             ignoreBom = true;
1078             scriptEncoding = contentCharset;
1079         }
1080         else {
1081             ignoreBom = true;
1082         }
1083 
1084         final String scriptCode = response.getContentAsString(scriptEncoding,
1085                                 ignoreBom
1086                                 && getWebClient().getBrowserVersion().hasFeature(JS_IGNORES_UTF8_BOM_SOMETIMES));
1087         if (null != scriptCode) {
1088             final AbstractJavaScriptEngine<?> javaScriptEngine = client.getJavaScriptEngine();
1089             final Object script = javaScriptEngine.compile(this, scriptCode, url.toExternalForm(), 1);
1090             if (script != null && cache.cacheIfPossible(request, response, script)) {
1091                 // no cleanup if the response is stored inside the cache
1092                 return script;
1093             }
1094 
1095             response.cleanUp();
1096             return script;
1097         }
1098 
1099         response.cleanUp();
1100         return null;
1101     }
1102 
1103     /**
1104      * Returns the title of this page or an empty string if the title wasn't specified.
1105      *
1106      * @return the title of this page or an empty string if the title wasn't specified
1107      */
1108     public String getTitleText() {
1109         final HtmlTitle titleElement = getTitleElement();
1110         if (titleElement != null) {
1111             return titleElement.asText();
1112         }
1113         return "";
1114     }
1115 
1116     /**
1117      * Sets the text for the title of this page. If there is not a title element
1118      * on this page, then one has to be generated.
1119      * @param message the new text
1120      */
1121     public void setTitleText(final String message) {
1122         HtmlTitle titleElement = getTitleElement();
1123         if (titleElement == null) {
1124             if (LOG.isDebugEnabled()) {
1125                 LOG.debug("No title element, creating one");
1126             }
1127             final HtmlHead head = (HtmlHead) getFirstChildElement(getDocumentElement(), HtmlHead.class);
1128             if (head == null) {
1129                 // perhaps should we create head too?
1130                 throw new IllegalStateException("Headelement was not defined for this page");
1131             }
1132             final Map<String, DomAttr> emptyMap = Collections.emptyMap();
1133             titleElement = new HtmlTitle(HtmlTitle.TAG_NAME, this, emptyMap);
1134             if (head.getFirstChild() != null) {
1135                 head.getFirstChild().insertBefore(titleElement);
1136             }
1137             else {
1138                 head.appendChild(titleElement);
1139             }
1140         }
1141 
1142         titleElement.setNodeValue(message);
1143     }
1144 
1145     /**
1146      * Gets the first child of startElement that is an instance of the given class.
1147      * @param startElement the parent element
1148      * @param clazz the class to search for
1149      * @return {@code null} if no child found
1150      */
1151     private static DomElement getFirstChildElement(final DomElement startElement, final Class<?> clazz) {
1152         if (startElement == null) {
1153             return null;
1154         }
1155         for (final DomElement element : startElement.getChildElements()) {
1156             if (clazz.isInstance(element)) {
1157                 return element;
1158             }
1159         }
1160 
1161         return null;
1162     }
1163 
1164     /**
1165      * Gets the first child of startElement or it's children that is an instance of the given class.
1166      * @param startElement the parent element
1167      * @param clazz the class to search for
1168      * @return {@code null} if no child found
1169      */
1170     private DomElement getFirstChildElementRecursive(final DomElement startElement, final Class<?> clazz) {
1171         if (startElement == null) {
1172             return null;
1173         }
1174         for (final DomElement element : startElement.getChildElements()) {
1175             if (clazz.isInstance(element)) {
1176                 return element;
1177             }
1178             final DomElement childFound = getFirstChildElementRecursive(element, clazz);
1179             if (childFound != null) {
1180                 return childFound;
1181             }
1182         }
1183 
1184         return null;
1185     }
1186 
1187     /**
1188      * Gets the title element for this page. Returns null if one is not found.
1189      *
1190      * @return the title element for this page or null if this is not one
1191      */
1192     private HtmlTitle getTitleElement() {
1193         return (HtmlTitle) getFirstChildElementRecursive(getDocumentElement(), HtmlTitle.class);
1194     }
1195 
1196     /**
1197      * Looks for and executes any appropriate event handlers. Looks for body and frame tags.
1198      * @param eventType either {@link Event#TYPE_LOAD}, {@link Event#TYPE_UNLOAD}, or {@link Event#TYPE_BEFORE_UNLOAD}
1199      * @return {@code true} if user accepted <tt>onbeforeunload</tt> (not relevant to other events)
1200      */
1201     private boolean executeEventHandlersIfNeeded(final String eventType) {
1202         // If JavaScript isn't enabled, there's nothing for us to do.
1203         if (!getWebClient().getOptions().isJavaScriptEnabled()) {
1204             return true;
1205         }
1206 
1207         // Execute the specified event on the document element.
1208         final WebWindow window = getEnclosingWindow();
1209         if (window.getScriptableObject() instanceof Window) {
1210             final DomElement element = getDocumentElement();
1211             if (element == null) { // happens for instance if document.documentElement has been removed from parent
1212                 return true;
1213             }
1214             final Event event;
1215             if (eventType.equals(Event.TYPE_BEFORE_UNLOAD)) {
1216                 event = new BeforeUnloadEvent(element, eventType);
1217             }
1218             else {
1219                 event = new Event(element, eventType);
1220             }
1221             final ScriptResult result = element.fireEvent(event);
1222             if (!isOnbeforeunloadAccepted(this, event, result)) {
1223                 return false;
1224             }
1225         }
1226 
1227         // If this page was loaded in a frame, execute the version of the event specified on the frame tag.
1228         if (window instanceof FrameWindow) {
1229             final FrameWindow fw = (FrameWindow) window;
1230             final BaseFrameElement frame = fw.getFrameElement();
1231 
1232             // if part of a document fragment, then the load event is not triggered
1233             if (Event.TYPE_LOAD.equals(eventType) && frame.getParentNode() instanceof DomDocumentFragment) {
1234                 return true;
1235             }
1236 
1237             if (frame.hasEventHandlers("on" + eventType)) {
1238                 if (LOG.isDebugEnabled()) {
1239                     LOG.debug("Executing on" + eventType + " handler for " + frame);
1240                 }
1241                 if (window.getScriptableObject() instanceof Window) {
1242                     final Event event;
1243                     if (eventType.equals(Event.TYPE_BEFORE_UNLOAD)) {
1244                         event = new BeforeUnloadEvent(frame, eventType);
1245                     }
1246                     else {
1247                         event = new Event(frame, eventType);
1248                     }
1249                     final ScriptResult result = ((Node) frame.getScriptableObject()).executeEventLocally(event);
1250                     if (!isOnbeforeunloadAccepted((HtmlPage) frame.getPage(), event, result)) {
1251                         return false;
1252                     }
1253                 }
1254             }
1255         }
1256 
1257         return true;
1258     }
1259 
1260     /**
1261      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
1262      *
1263      * @return true if the OnbeforeunloadHandler has accepted to change the page
1264      */
1265     public boolean isOnbeforeunloadAccepted() {
1266         return executeEventHandlersIfNeeded(Event.TYPE_BEFORE_UNLOAD);
1267     }
1268 
1269     private boolean isOnbeforeunloadAccepted(final HtmlPage page, final Event event, final ScriptResult result) {
1270         if (event.getType().equals(Event.TYPE_BEFORE_UNLOAD)) {
1271             final boolean ie = hasFeature(JS_CALL_RESULT_IS_LAST_RETURN_VALUE);
1272             final String message = getBeforeUnloadMessage(event, result, ie);
1273             if (message != null) {
1274                 final OnbeforeunloadHandler handler = getWebClient().getOnbeforeunloadHandler();
1275                 if (handler == null) {
1276                     LOG.warn("document.onbeforeunload() returned a string in event.returnValue,"
1277                             + " but no onbeforeunload handler installed.");
1278                 }
1279                 else {
1280                     return handler.handleEvent(page, message);
1281                 }
1282             }
1283         }
1284         return true;
1285     }
1286 
1287     private static String getBeforeUnloadMessage(final Event event, final ScriptResult result, final boolean ie) {
1288         String message = null;
1289         if (event.getReturnValue() != Undefined.instance) {
1290             if (!ie || event.getReturnValue() != null || result == null || result.getJavaScriptResult() == null
1291                     || result.getJavaScriptResult() == Undefined.instance) {
1292                 message = Context.toString(event.getReturnValue());
1293             }
1294         }
1295         else {
1296             if (result != null) {
1297                 if (ie) {
1298                     if (result.getJavaScriptResult() != Undefined.instance) {
1299                         message = Context.toString(result.getJavaScriptResult());
1300                     }
1301                 }
1302                 else if (result.getJavaScriptResult() != null
1303                         && result.getJavaScriptResult() != Undefined.instance) {
1304                     message = Context.toString(result.getJavaScriptResult());
1305                 }
1306             }
1307         }
1308         return message;
1309     }
1310 
1311     /**
1312      * If a refresh has been specified either through a meta tag or an HTTP
1313      * response header, then perform that refresh.
1314      * @throws IOException if an IO problem occurs
1315      */
1316     private void executeRefreshIfNeeded() throws IOException {
1317         // If this page is not in a frame then a refresh has already happened,
1318         // most likely through the JavaScript onload handler, so we don't do a
1319         // second refresh.
1320         final WebWindow window = getEnclosingWindow();
1321         if (window == null) {
1322             return;
1323         }
1324 
1325         final String refreshString = getRefreshStringOrNull();
1326         if (refreshString == null || refreshString.isEmpty()) {
1327             return;
1328         }
1329 
1330         final double time;
1331         final URL url;
1332 
1333         int index = StringUtils.indexOfAnyBut(refreshString, "0123456789");
1334         final boolean timeOnly = index == -1;
1335 
1336         if (timeOnly) {
1337             // Format: <meta http-equiv='refresh' content='10'>
1338             try {
1339                 time = Double.parseDouble(refreshString);
1340             }
1341             catch (final NumberFormatException e) {
1342                 LOG.error("Malformed refresh string (no ';' but not a number): " + refreshString, e);
1343                 return;
1344             }
1345             url = getUrl();
1346         }
1347         else {
1348             // Format: <meta http-equiv='refresh' content='10;url=http://www.blah.com'>
1349             try {
1350                 time = Double.parseDouble(refreshString.substring(0, index).trim());
1351             }
1352             catch (final NumberFormatException e) {
1353                 LOG.error("Malformed refresh string (no valid number before ';') " + refreshString, e);
1354                 return;
1355             }
1356             index = refreshString.toLowerCase(Locale.ROOT).indexOf("url=", index);
1357             if (index == -1) {
1358                 LOG.error("Malformed refresh string (found ';' but no 'url='): " + refreshString);
1359                 return;
1360             }
1361             final StringBuilder builder = new StringBuilder(refreshString.substring(index + 4));
1362             if (StringUtils.isBlank(builder.toString())) {
1363                 //content='10; URL=' is treated as content='10'
1364                 url = getUrl();
1365             }
1366             else {
1367                 if (builder.charAt(0) == '"' || builder.charAt(0) == 0x27) {
1368                     builder.deleteCharAt(0);
1369                 }
1370                 if (builder.charAt(builder.length() - 1) == '"' || builder.charAt(builder.length() - 1) == 0x27) {
1371                     builder.deleteCharAt(builder.length() - 1);
1372                 }
1373                 final String urlString = builder.toString();
1374                 try {
1375                     url = getFullyQualifiedUrl(urlString);
1376                 }
1377                 catch (final MalformedURLException e) {
1378                     LOG.error("Malformed URL in refresh string: " + refreshString, e);
1379                     throw e;
1380                 }
1381             }
1382         }
1383 
1384         final int timeRounded = (int) time;
1385         checkRecursion();
1386         getWebClient().getRefreshHandler().handleRefresh(this, url, timeRounded);
1387     }
1388 
1389     private void checkRecursion() {
1390         final StackTraceElement[] elements = new Exception().getStackTrace();
1391         if (elements.length > 500) {
1392             for (int i = 0; i < 500; i++) {
1393                 if (!elements[i].getClassName().startsWith("com.gargoylesoftware.htmlunit.")) {
1394                     return;
1395                 }
1396             }
1397             final WebResponse webResponse = getWebResponse();
1398             throw new FailingHttpStatusCodeException("Too much redirect for "
1399                     + webResponse.getWebRequest().getUrl(), webResponse);
1400         }
1401     }
1402 
1403     /**
1404      * Returns an auto-refresh string if specified. This will look in both the meta
1405      * tags and inside the HTTP response headers.
1406      * @return the auto-refresh string
1407      */
1408     private String getRefreshStringOrNull() {
1409         final List<HtmlMeta> metaTags = getMetaTags("refresh");
1410         if (!metaTags.isEmpty()) {
1411             return metaTags.get(0).getContentAttribute().trim();
1412         }
1413         return getWebResponse().getResponseHeaderValue("Refresh");
1414     }
1415 
1416     /**
1417      * Executes any deferred scripts, if necessary.
1418      */
1419     private void executeDeferredScriptsIfNeeded() {
1420         if (!getWebClient().getOptions().isJavaScriptEnabled()) {
1421             return;
1422         }
1423         if (hasFeature(JS_DEFERRED)) {
1424             final DomElement doc = getDocumentElement();
1425             final List<HtmlElement> elements = doc.getElementsByTagName("script");
1426             for (final HtmlElement e : elements) {
1427                 if (e instanceof HtmlScript) {
1428                     final HtmlScript script = (HtmlScript) e;
1429                     if (script.isDeferred()) {
1430                         script.executeScriptIfNeeded();
1431                     }
1432                 }
1433             }
1434         }
1435     }
1436 
1437     /**
1438      * Sets the ready state on any deferred scripts, if necessary.
1439      */
1440     private void setReadyStateOnDeferredScriptsIfNeeded() {
1441         if (getWebClient().getOptions().isJavaScriptEnabled() && hasFeature(JS_DEFERRED)) {
1442             final List<HtmlElement> elements = getDocumentElement().getElementsByTagName("script");
1443             for (final HtmlElement e : elements) {
1444                 if (e instanceof HtmlScript) {
1445                     final HtmlScript script = (HtmlScript) e;
1446                     if (script.isDeferred()) {
1447                         script.setAndExecuteReadyState(READY_STATE_COMPLETE);
1448                     }
1449                 }
1450             }
1451         }
1452     }
1453 
1454     /**
1455      * Deregister frames that are no longer in use.
1456      */
1457     public void deregisterFramesIfNeeded() {
1458         for (final WebWindow window : getFrames()) {
1459             getWebClient().deregisterWebWindow(window);
1460             final Page page = window.getEnclosedPage();
1461             if (page != null && page.isHtmlPage()) {
1462                 // seems quite silly, but for instance if the src attribute of an iframe is not
1463                 // set, the error only occurs when leaving the page
1464                 ((HtmlPage) page).deregisterFramesIfNeeded();
1465             }
1466         }
1467     }
1468 
1469     /**
1470      * Returns a list containing all the frames (from frame and iframe tags) in this page.
1471      * @return a list of {@link FrameWindow}
1472      */
1473     public List<FrameWindow> getFrames() {
1474         final List<FrameWindow> list = new ArrayList<>(frameElements_.size());
1475         for (final BaseFrameElement frameElement : frameElements_) {
1476             list.add(frameElement.getEnclosedWindow());
1477         }
1478         return list;
1479     }
1480 
1481     /**
1482      * Returns the first frame contained in this page with the specified name.
1483      * @param name the name to search for
1484      * @return the first frame found
1485      * @exception ElementNotFoundException If no frame exist in this page with the specified name.
1486      */
1487     public FrameWindow getFrameByName(final String name) throws ElementNotFoundException {
1488         for (final FrameWindow frame : getFrames()) {
1489             if (frame.getName().equals(name)) {
1490                 return frame;
1491             }
1492         }
1493 
1494         throw new ElementNotFoundException("frame or iframe", "name", name);
1495     }
1496 
1497     /**
1498      * Simulate pressing an access key. This may change the focus, may click buttons and may invoke
1499      * JavaScript.
1500      *
1501      * @param accessKey the key that will be pressed
1502      * @return the element that has the focus after pressing this access key or null if no element
1503      * has the focus.
1504      * @throws IOException if an IO error occurs during the processing of this access key (this
1505      *         would only happen if the access key triggered a button which in turn caused a page load)
1506      */
1507     public DomElement pressAccessKey(final char accessKey) throws IOException {
1508         final HtmlElement element = getHtmlElementByAccessKey(accessKey);
1509         if (element != null) {
1510             element.focus();
1511             final Page newPage;
1512             if (element instanceof HtmlAnchor || element instanceof HtmlArea || element instanceof HtmlButton
1513                     || element instanceof HtmlInput || element instanceof HtmlLabel || element instanceof HtmlLegend
1514                     || element instanceof HtmlTextArea || element instanceof HtmlArea) {
1515                 newPage = element.click();
1516             }
1517             else {
1518                 newPage = this;
1519             }
1520 
1521             if (newPage != this && getFocusedElement() == element) {
1522                 // The page was reloaded therefore no element on this page will have the focus.
1523                 getFocusedElement().blur();
1524             }
1525         }
1526 
1527         return getFocusedElement();
1528     }
1529 
1530     /**
1531      * Move the focus to the next element in the tab order. To determine the specified tab
1532      * order, refer to {@link HtmlPage#getTabbableElements()}
1533      *
1534      * @return the element that has focus after calling this method
1535      */
1536     public HtmlElement tabToNextElement() {
1537         final List<HtmlElement> elements = getTabbableElements();
1538         if (elements.isEmpty()) {
1539             setFocusedElement(null);
1540             return null;
1541         }
1542 
1543         final HtmlElement elementToGiveFocus;
1544         final DomElement elementWithFocus = getFocusedElement();
1545         if (elementWithFocus == null) {
1546             elementToGiveFocus = elements.get(0);
1547         }
1548         else {
1549             final int index = elements.indexOf(elementWithFocus);
1550             if (index == -1) {
1551                 // The element with focus isn't on this page
1552                 elementToGiveFocus = elements.get(0);
1553             }
1554             else {
1555                 if (index == elements.size() - 1) {
1556                     elementToGiveFocus = elements.get(0);
1557                 }
1558                 else {
1559                     elementToGiveFocus = elements.get(index + 1);
1560                 }
1561             }
1562         }
1563 
1564         setFocusedElement(elementToGiveFocus);
1565         return elementToGiveFocus;
1566     }
1567 
1568     /**
1569      * Move the focus to the previous element in the tab order. To determine the specified tab
1570      * order, refer to {@link HtmlPage#getTabbableElements()}
1571      *
1572      * @return the element that has focus after calling this method
1573      */
1574     public HtmlElement tabToPreviousElement() {
1575         final List<HtmlElement> elements = getTabbableElements();
1576         if (elements.isEmpty()) {
1577             setFocusedElement(null);
1578             return null;
1579         }
1580 
1581         final HtmlElement elementToGiveFocus;
1582         final DomElement elementWithFocus = getFocusedElement();
1583         if (elementWithFocus == null) {
1584             elementToGiveFocus = elements.get(elements.size() - 1);
1585         }
1586         else {
1587             final int index = elements.indexOf(elementWithFocus);
1588             if (index == -1) {
1589                 // The element with focus isn't on this page
1590                 elementToGiveFocus = elements.get(elements.size() - 1);
1591             }
1592             else {
1593                 if (index == 0) {
1594                     elementToGiveFocus = elements.get(elements.size() - 1);
1595                 }
1596                 else {
1597                     elementToGiveFocus = elements.get(index - 1);
1598                 }
1599             }
1600         }
1601 
1602         setFocusedElement(elementToGiveFocus);
1603         return elementToGiveFocus;
1604     }
1605 
1606     /**
1607      * Returns the HTML element with the specified ID. If more than one element
1608      * has this ID (not allowed by the HTML spec), then this method returns the
1609      * first one.
1610      *
1611      * @param elementId the ID value to search for
1612      * @param <E> the element type
1613      * @return the HTML element with the specified ID
1614      * @throws ElementNotFoundException if no element was found matching the specified ID
1615      */
1616     @SuppressWarnings("unchecked")
1617     public <E extends HtmlElement> E getHtmlElementById(final String elementId) throws ElementNotFoundException {
1618         final DomElement element = getElementById(elementId);
1619         if (element == null) {
1620             throw new ElementNotFoundException("*", "id", elementId);
1621         }
1622         return (E) element;
1623     }
1624 
1625     /**
1626      * Returns the elements with the specified ID. If there are no elements
1627      * with the specified ID, this method returns an empty list. Please note that
1628      * the lists returned by this method are immutable.
1629      *
1630      * @param elementId the ID value to search for
1631      * @return the elements with the specified name attribute
1632      */
1633     public List<DomElement> getElementsById(final String elementId) {
1634         final SortedSet<DomElement> elements = idMap_.get(elementId);
1635         if (elements != null) {
1636             return new ArrayList<>(elements);
1637         }
1638         return Collections.emptyList();
1639     }
1640 
1641     /**
1642      * Returns the element with the specified name. If more than one element
1643      * has this name, then this method returns the first one.
1644      *
1645      * @param name the name value to search for
1646      * @param <E> the element type
1647      * @return the element with the specified name
1648      * @throws ElementNotFoundException if no element was found matching the specified name
1649      */
1650     @SuppressWarnings("unchecked")
1651     public <E extends DomElement> E getElementByName(final String name) throws ElementNotFoundException {
1652         final SortedSet<DomElement> elements = nameMap_.get(name);
1653         if (elements != null) {
1654             return (E) elements.first();
1655         }
1656         throw new ElementNotFoundException("*", "name", name);
1657     }
1658 
1659     /**
1660      * Returns the elements with the specified name attribute. If there are no elements
1661      * with the specified name, this method returns an empty list. Please note that
1662      * the lists returned by this method are immutable.
1663      *
1664      * @param name the name value to search for
1665      * @return the elements with the specified name attribute
1666      */
1667     public List<DomElement> getElementsByName(final String name) {
1668         final SortedSet<DomElement> elements = nameMap_.get(name);
1669         if (elements != null) {
1670             return new ArrayList<>(elements);
1671         }
1672         return Collections.emptyList();
1673     }
1674 
1675     /**
1676      * Returns the elements with the specified string for their name or ID. If there are
1677      * no elements with the specified name or ID, this method returns an empty list.
1678      *
1679      * @param idAndOrName the value to search for
1680      * @return the elements with the specified string for their name or ID
1681      */
1682     public List<DomElement> getElementsByIdAndOrName(final String idAndOrName) {
1683         final Collection<DomElement> list1 = idMap_.get(idAndOrName);
1684         final Collection<DomElement> list2 = nameMap_.get(idAndOrName);
1685         final List<DomElement> list = new ArrayList<>();
1686         if (list1 != null) {
1687             list.addAll(list1);
1688         }
1689         if (list2 != null) {
1690             for (final DomElement elt : list2) {
1691                 if (!list.contains(elt)) {
1692                     list.add(elt);
1693                 }
1694             }
1695         }
1696         return list;
1697     }
1698 
1699     /**
1700      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
1701      *
1702      * @param node the node that has just been added to the document
1703      */
1704     void notifyNodeAdded(final DomNode node) {
1705         if (node instanceof DomElement) {
1706             addMappedElement((DomElement) node, true);
1707 
1708             if (node instanceof BaseFrameElement) {
1709                 frameElements_.add((BaseFrameElement) node);
1710             }
1711             for (final HtmlElement child : node.getHtmlElementDescendants()) {
1712                 if (child instanceof BaseFrameElement) {
1713                     frameElements_.add((BaseFrameElement) child);
1714                 }
1715             }
1716 
1717             if ("base".equals(node.getNodeName())) {
1718                 calculateBase();
1719             }
1720         }
1721         node.onAddedToPage();
1722     }
1723 
1724     /**
1725      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
1726      *
1727      * @param node the node that has just been removed from the tree
1728      */
1729     void notifyNodeRemoved(final DomNode node) {
1730         if (node instanceof HtmlElement) {
1731             removeMappedElement((HtmlElement) node, true, true);
1732 
1733             if (node instanceof BaseFrameElement) {
1734                 frameElements_.remove(node);
1735             }
1736             for (final HtmlElement child : node.getHtmlElementDescendants()) {
1737                 if (child instanceof BaseFrameElement) {
1738                     frameElements_.remove(child);
1739                 }
1740             }
1741 
1742             if ("base".equals(node.getNodeName())) {
1743                 calculateBase();
1744             }
1745         }
1746     }
1747 
1748     /**
1749      * Adds an element to the ID and name maps, if necessary.
1750      * @param element the element to be added to the ID and name maps
1751      */
1752     void addMappedElement(final DomElement element) {
1753         addMappedElement(element, false);
1754     }
1755 
1756     /**
1757      * Adds an element to the ID and name maps, if necessary.
1758      * @param element the element to be added to the ID and name maps
1759      * @param recurse indicates if children must be added too
1760      */
1761     void addMappedElement(final DomElement element, final boolean recurse) {
1762         if (isAncestorOf(element)) {
1763             addElement(idMap_, element, "id", recurse);
1764             addElement(nameMap_, element, "name", recurse);
1765         }
1766     }
1767 
1768     private void addElement(final Map<String, SortedSet<DomElement>> map, final DomElement element,
1769             final String attribute, final boolean recurse) {
1770         final String value = getAttributeValue(element, attribute);
1771 
1772         if (DomElement.ATTRIBUTE_NOT_DEFINED != value) {
1773             SortedSet<DomElement> elements = map.get(value);
1774             if (elements == null) {
1775                 elements = new TreeSet<>(documentPositionComparator);
1776                 elements.add(element);
1777                 map.put(value, elements);
1778             }
1779             else if (!elements.contains(element)) {
1780                 elements.add(element);
1781             }
1782         }
1783         if (recurse) {
1784             for (final DomElement child : element.getChildElements()) {
1785                 addElement(map, child, attribute, true);
1786             }
1787         }
1788     }
1789 
1790     private static String getAttributeValue(final DomElement element, final String attribute) {
1791         // first try real attributes
1792         String value = element.getAttribute(attribute);
1793 
1794         if (DomElement.ATTRIBUTE_NOT_DEFINED == value
1795                 && !(element instanceof HtmlApplet)
1796                 && !(element instanceof HtmlObject)) {
1797             // second try are JavaScript attributes
1798             // ...but applets/objects are a bit special so ignore them
1799             final Object o = element.getScriptableObject();
1800             if (o instanceof ScriptableObject) {
1801                 final ScriptableObject scriptObject = (ScriptableObject) o;
1802                 // we have to make sure the scriptObject has a slot for the given attribute.
1803                 // just using get() may use e.g. getWithPreemption().
1804                 if (scriptObject.has(attribute, scriptObject)) {
1805                     final Object jsValue = scriptObject.get(attribute, scriptObject);
1806                     if (jsValue != Scriptable.NOT_FOUND && jsValue instanceof String) {
1807                         value = (String) jsValue;
1808                     }
1809                 }
1810             }
1811         }
1812         return value;
1813     }
1814 
1815     /**
1816      * Removes an element from the ID and name maps, if necessary.
1817      * @param element the element to be removed from the ID and name maps
1818      */
1819     void removeMappedElement(final HtmlElement element) {
1820         removeMappedElement(element, false, false);
1821     }
1822 
1823     /**
1824      * Removes an element and optionally its children from the ID and name maps, if necessary.
1825      * @param element the element to be removed from the ID and name maps
1826      * @param recurse indicates if children must be removed too
1827      * @param descendant indicates of the element was descendant of this HtmlPage, but now its parent might be null
1828      */
1829     void removeMappedElement(final DomElement element, final boolean recurse, final boolean descendant) {
1830         if (descendant || isAncestorOf(element)) {
1831             removeElement(idMap_, element, "id", recurse);
1832             removeElement(nameMap_, element, "name", recurse);
1833         }
1834     }
1835 
1836     private void removeElement(final Map<String, SortedSet<DomElement>> map, final DomElement element,
1837             final String attribute, final boolean recurse) {
1838         final String value = getAttributeValue(element, attribute);
1839 
1840         if (DomElement.ATTRIBUTE_NOT_DEFINED != value) {
1841             final SortedSet<DomElement> elements = map.remove(value);
1842             if (elements != null && (elements.size() != 1 || !elements.contains(element))) {
1843                 elements.remove(element);
1844                 map.put(value, elements);
1845             }
1846         }
1847         if (recurse) {
1848             for (final DomElement child : element.getChildElements()) {
1849                 removeElement(map, child, attribute, true);
1850             }
1851         }
1852     }
1853 
1854     /**
1855      * Indicates if the attribute name indicates that the owning element is mapped.
1856      * @param document the owning document
1857      * @param attributeName the name of the attribute to consider
1858      * @return {@code true} if the owning element should be mapped in its owning page
1859      */
1860     static boolean isMappedElement(final Document document, final String attributeName) {
1861         return document instanceof HtmlPage
1862             && ("name".equals(attributeName) || "id".equals(attributeName));
1863     }
1864 
1865     private void calculateBase() {
1866         final List<HtmlElement> baseElements = getDocumentElement().getElementsByTagName("base");
1867         switch (baseElements.size()) {
1868             case 0:
1869                 base_ = null;
1870                 break;
1871 
1872             case 1:
1873                 base_ = (HtmlBase) baseElements.get(0);
1874                 break;
1875 
1876             default:
1877                 base_ = (HtmlBase) baseElements.get(0);
1878                 notifyIncorrectness("Multiple 'base' detected, only the first is used.");
1879         }
1880     }
1881 
1882     /**
1883      * Loads the content of the contained frames. This is done after the page is completely loaded, to allow script
1884      * contained in the frames to reference elements from the page located after the closing &lt;/frame&gt; tag.
1885      * @throws FailingHttpStatusCodeException if the server returns a failing status code AND the property
1886      *         {@link WebClient#setThrowExceptionOnFailingStatusCode(boolean)} is set to {@code true}
1887      */
1888     void loadFrames() throws FailingHttpStatusCodeException {
1889         for (final FrameWindow w : getFrames()) {
1890             final BaseFrameElement frame = w.getFrameElement();
1891             // test if the frame should really be loaded:
1892             // if a script has already changed its content, it should be skipped
1893             // use == and not equals(...) to identify initial content (versus URL set to "about:blank")
1894             if (frame.getEnclosedWindow() != null
1895                     && WebClient.URL_ABOUT_BLANK == frame.getEnclosedPage().getUrl()
1896                     && !frame.isContentLoaded()) {
1897                 frame.loadInnerPage();
1898             }
1899         }
1900     }
1901 
1902     /**
1903      * Gives a basic representation for debugging purposes.
1904      * @return a basic representation
1905      */
1906     @Override
1907     public String toString() {
1908         final StringBuilder builder = new StringBuilder();
1909         builder.append("HtmlPage(");
1910         builder.append(getUrl());
1911         builder.append(")@");
1912         builder.append(hashCode());
1913         return builder.toString();
1914     }
1915 
1916     /**
1917      * Gets the meta tag for a given {@code http-equiv} value.
1918      * @param httpEquiv the {@code http-equiv} value
1919      * @return a list of {@link HtmlMeta}
1920      */
1921     protected List<HtmlMeta> getMetaTags(final String httpEquiv) {
1922         if (getDocumentElement() == null) {
1923             return Collections.emptyList(); // weird case, for instance if document.documentElement has been removed
1924         }
1925         final String nameLC = httpEquiv.toLowerCase(Locale.ROOT);
1926         final List<HtmlMeta> tags = getDocumentElement().getElementsByTagNameImpl("meta");
1927         final List<HtmlMeta> foundTags = new ArrayList<>();
1928         for (HtmlMeta htmlMeta : tags) {
1929             if (nameLC.equals(htmlMeta.getHttpEquivAttribute().toLowerCase(Locale.ROOT))) {
1930                 foundTags.add(htmlMeta);
1931             }
1932         }
1933         return foundTags;
1934     }
1935 
1936     /**
1937      * Creates a clone of this instance, and clears cached state to be not shared with the original.
1938      *
1939      * @return a clone of this instance
1940      */
1941     @Override
1942     protected HtmlPage clone() {
1943         final HtmlPage result = (HtmlPage) super.clone();
1944         result.elementWithFocus_ = null;
1945 
1946         result.idMap_ = Collections.synchronizedMap(new HashMap<String, SortedSet<DomElement>>());
1947         result.nameMap_ = Collections.synchronizedMap(new HashMap<String, SortedSet<DomElement>>());
1948 
1949         return result;
1950     }
1951 
1952     /**
1953      * {@inheritDoc}
1954      */
1955     @Override
1956     public HtmlPage cloneNode(final boolean deep) {
1957         // we need the ScriptObject clone before cloning the kids.
1958         final HtmlPage result = (HtmlPage) super.cloneNode(false);
1959         final SimpleScriptable jsObjClone = ((SimpleScriptable) getScriptableObject()).clone();
1960         jsObjClone.setDomNode(result);
1961 
1962         // if deep, clone the kids too, and re initialize parts of the clone
1963         if (deep) {
1964             synchronized (lock_) {
1965                 result.attributeListeners_ = null;
1966             }
1967             result.selectionRanges_ = new ArrayList<>(3);
1968             result.afterLoadActions_ = new ArrayList<>();
1969             result.frameElements_ = new TreeSet<>(documentPositionComparator);
1970             for (DomNode child = getFirstChild(); child != null; child = child.getNextSibling()) {
1971                 result.appendChild(child.cloneNode(true));
1972             }
1973         }
1974         return result;
1975     }
1976 
1977     /**
1978      * Adds an HtmlAttributeChangeListener to the listener list.
1979      * The listener is registered for all attributes of all HtmlElements contained in this page.
1980      *
1981      * @param listener the attribute change listener to be added
1982      * @see #removeHtmlAttributeChangeListener(HtmlAttributeChangeListener)
1983      */
1984     public void addHtmlAttributeChangeListener(final HtmlAttributeChangeListener listener) {
1985         WebAssert.notNull("listener", listener);
1986         synchronized (lock_) {
1987             if (attributeListeners_ == null) {
1988                 attributeListeners_ = new LinkedHashSet<>();
1989             }
1990             attributeListeners_.add(listener);
1991         }
1992     }
1993 
1994     /**
1995      * Removes an HtmlAttributeChangeListener from the listener list.
1996      * This method should be used to remove HtmlAttributeChangeListener that were registered
1997      * for all attributes of all HtmlElements contained in this page.
1998      *
1999      * @param listener the attribute change listener to be removed
2000      * @see #addHtmlAttributeChangeListener(HtmlAttributeChangeListener)
2001      */
2002     public void removeHtmlAttributeChangeListener(final HtmlAttributeChangeListener listener) {
2003         WebAssert.notNull("listener", listener);
2004         synchronized (lock_) {
2005             if (attributeListeners_ != null) {
2006                 attributeListeners_.remove(listener);
2007             }
2008         }
2009     }
2010 
2011     /**
2012      * Notifies all registered listeners for the given event to add an attribute.
2013      * @param event the event to fire
2014      */
2015     void fireHtmlAttributeAdded(final HtmlAttributeChangeEvent event) {
2016         final List<HtmlAttributeChangeListener> listeners = safeGetAttributeListeners();
2017         if (listeners != null) {
2018             for (final HtmlAttributeChangeListener listener : listeners) {
2019                 listener.attributeAdded(event);
2020             }
2021         }
2022     }
2023 
2024     /**
2025      * Notifies all registered listeners for the given event to replace an attribute.
2026      * @param event the event to fire
2027      */
2028     void fireHtmlAttributeReplaced(final HtmlAttributeChangeEvent event) {
2029         final List<HtmlAttributeChangeListener> listeners = safeGetAttributeListeners();
2030         if (listeners != null) {
2031             for (final HtmlAttributeChangeListener listener : listeners) {
2032                 listener.attributeReplaced(event);
2033             }
2034         }
2035     }
2036 
2037     /**
2038      * Notifies all registered listeners for the given event to remove an attribute.
2039      * @param event the event to fire
2040      */
2041     void fireHtmlAttributeRemoved(final HtmlAttributeChangeEvent event) {
2042         final List<HtmlAttributeChangeListener> listeners = safeGetAttributeListeners();
2043         if (listeners != null) {
2044             for (final HtmlAttributeChangeListener listener : listeners) {
2045                 listener.attributeRemoved(event);
2046             }
2047         }
2048     }
2049 
2050     private List<HtmlAttributeChangeListener> safeGetAttributeListeners() {
2051         synchronized (lock_) {
2052             if (attributeListeners_ != null) {
2053                 return new ArrayList<>(attributeListeners_);
2054             }
2055             return null;
2056         }
2057     }
2058 
2059     /**
2060      * {@inheritDoc}
2061      */
2062     @Override
2063     protected void checkChildHierarchy(final org.w3c.dom.Node newChild) throws DOMException {
2064         if (newChild instanceof Element) {
2065             if (getDocumentElement() != null) {
2066                 throw new DOMException(DOMException.HIERARCHY_REQUEST_ERR,
2067                     "The Document may only have a single child Element.");
2068             }
2069         }
2070         else if (newChild instanceof DocumentType) {
2071             if (getDoctype() != null) {
2072                 throw new DOMException(DOMException.HIERARCHY_REQUEST_ERR,
2073                     "The Document may only have a single child DocumentType.");
2074             }
2075         }
2076         else if (!(newChild instanceof Comment || newChild instanceof ProcessingInstruction)) {
2077             throw new DOMException(DOMException.HIERARCHY_REQUEST_ERR,
2078                 "The Document may not have a child of this type: " + newChild.getNodeType());
2079         }
2080         super.checkChildHierarchy(newChild);
2081     }
2082 
2083     /**
2084      * Returns {@code true} if an HTML parser is operating on this page, adding content to it.
2085      * @return {@code true} if an HTML parser is operating on this page, adding content to it
2086      */
2087     public boolean isBeingParsed() {
2088         return parserCount_ > 0;
2089     }
2090 
2091     /**
2092      * Called by the HTML parser to let the page know that it has started parsing some content for this page.
2093      */
2094     void registerParsingStart() {
2095         parserCount_++;
2096     }
2097 
2098     /**
2099      * Called by the HTML parser to let the page know that it has finished parsing some content for this page.
2100      */
2101     void registerParsingEnd() {
2102         parserCount_--;
2103     }
2104 
2105     /**
2106      * Returns {@code true} if an HTML parser is parsing a non-inline HTML snippet to add content
2107      * to this page. Non-inline content is content that is parsed for the page, but not in the
2108      * same stream as the page itself -- basically anything other than <tt>document.write()</tt>
2109      * or <tt>document.writeln()</tt>: <tt>innerHTML</tt>, <tt>outerHTML</tt>,
2110      * <tt>document.createElement()</tt>, etc.
2111      *
2112      * @return {@code true} if an HTML parser is parsing a non-inline HTML snippet to add content
2113      *         to this page
2114      */
2115     boolean isParsingHtmlSnippet() {
2116         return snippetParserCount_ > 0;
2117     }
2118 
2119     /**
2120      * Called by the HTML parser to let the page know that it has started parsing a non-inline HTML snippet.
2121      */
2122     void registerSnippetParsingStart() {
2123         snippetParserCount_++;
2124     }
2125 
2126     /**
2127      * Called by the HTML parser to let the page know that it has finished parsing a non-inline HTML snippet.
2128      */
2129     void registerSnippetParsingEnd() {
2130         snippetParserCount_--;
2131     }
2132 
2133     /**
2134      * Returns {@code true} if an HTML parser is parsing an inline HTML snippet to add content
2135      * to this page. Inline content is content inserted into the parser stream dynamically
2136      * while the page is being parsed (i.e. <tt>document.write()</tt> or <tt>document.writeln()</tt>).
2137      *
2138      * @return {@code true} if an HTML parser is parsing an inline HTML snippet to add content
2139      *         to this page
2140      */
2141     boolean isParsingInlineHtmlSnippet() {
2142         return inlineSnippetParserCount_ > 0;
2143     }
2144 
2145     /**
2146      * Called by the HTML parser to let the page know that it has started parsing an inline HTML snippet.
2147      */
2148     void registerInlineSnippetParsingStart() {
2149         inlineSnippetParserCount_++;
2150     }
2151 
2152     /**
2153      * Called by the HTML parser to let the page know that it has finished parsing an inline HTML snippet.
2154      */
2155     void registerInlineSnippetParsingEnd() {
2156         inlineSnippetParserCount_--;
2157     }
2158 
2159     /**
2160      * Refreshes the page by sending the same parameters as previously sent to get this page.
2161      * @return the newly loaded page.
2162      * @throws IOException if an IO problem occurs
2163      */
2164     public Page refresh() throws IOException {
2165         return getWebClient().getPage(getWebResponse().getWebRequest());
2166     }
2167 
2168     /**
2169      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2170      * <p>
2171      * Parses the given string as would it belong to the content being parsed
2172      * at the current parsing position
2173      * </p>
2174      * @param string the HTML code to write in place
2175      */
2176     public void writeInParsedStream(final String string) {
2177         builder_.pushInputString(string);
2178     }
2179 
2180     /**
2181      * Sets the builder to allow page to send content from document.write(ln) calls.
2182      * @param htmlUnitDOMBuilder the builder
2183      */
2184     void setBuilder(final HtmlUnitDOMBuilder htmlUnitDOMBuilder) {
2185         builder_ = htmlUnitDOMBuilder;
2186     }
2187 
2188     /**
2189      * Returns the current builder.
2190      * @return the current builder
2191      */
2192     HtmlUnitDOMBuilder getBuilder() {
2193         return builder_;
2194     }
2195 
2196     /**
2197      * <p>Returns all namespaces defined in the root element of this page.</p>
2198      * <p>The default namespace has a key of an empty string.</p>
2199      * @return all namespaces defined in the root element of this page
2200      */
2201     public Map<String, String> getNamespaces() {
2202         final org.w3c.dom.NamedNodeMap attributes = getDocumentElement().getAttributes();
2203         final Map<String, String> namespaces = new HashMap<>();
2204         for (int i = 0; i < attributes.getLength(); i++) {
2205             final Attr attr = (Attr) attributes.item(i);
2206             String name = attr.getName();
2207             if (name.startsWith("xmlns")) {
2208                 int startPos = 5;
2209                 if (name.length() > 5 && name.charAt(5) == ':') {
2210                     startPos = 6;
2211                 }
2212                 name = name.substring(startPos);
2213                 namespaces.put(name, attr.getValue());
2214             }
2215         }
2216         return namespaces;
2217     }
2218 
2219     /**
2220      * {@inheritDoc}
2221      */
2222     @Override
2223     protected void setDocumentType(final DocumentType type) {
2224         super.setDocumentType(type);
2225     }
2226 
2227     /**
2228      * Saves the current page, with all images, to the specified location.
2229      * The default behavior removes all script elements.
2230      *
2231      * @param file file to write this page into
2232      * @throws IOException If an error occurs
2233      */
2234     public void save(final File file) throws IOException {
2235         new XmlSerializer().save(this, file);
2236     }
2237 
2238     /**
2239      * Returns whether the current page mode is in {@code quirks mode} or in {@code standards mode}.
2240      * @return true for {@code quirks mode}, false for {@code standards mode}
2241      */
2242     public boolean isQuirksMode() {
2243         return "BackCompat".equals(((HTMLDocument) getScriptableObject()).getCompatMode());
2244     }
2245 
2246     /**
2247      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2248      * {@inheritDoc}
2249      */
2250     @Override
2251     public boolean isAttachedToPage() {
2252         return true;
2253     }
2254 
2255     /**
2256      * {@inheritDoc}
2257      */
2258     @Override
2259     public boolean isHtmlPage() {
2260         return true;
2261     }
2262 
2263     /**
2264      * The base URL used to resolve relative URLs.
2265      * @return the base URL
2266      */
2267     public URL getBaseURL() {
2268         URL baseUrl;
2269         if (base_ == null) {
2270             baseUrl = getUrl();
2271             final WebWindow window = getEnclosingWindow();
2272             final boolean frame = window != window.getTopWindow();
2273             if (frame) {
2274                 final boolean frameSrcIsNotSet = baseUrl == WebClient.URL_ABOUT_BLANK;
2275                 final boolean frameSrcIsJs = "javascript".equals(baseUrl.getProtocol());
2276                 if (frameSrcIsNotSet || frameSrcIsJs) {
2277                     baseUrl = ((HtmlPage) window.getTopWindow().getEnclosedPage()).getWebResponse()
2278                         .getWebRequest().getUrl();
2279                 }
2280             }
2281             else if (baseUrl_ != null) {
2282                 baseUrl = baseUrl_;
2283             }
2284         }
2285         else {
2286             final String href = base_.getHrefAttribute().trim();
2287             if (StringUtils.isEmpty(href)) {
2288                 baseUrl = getUrl();
2289             }
2290             else {
2291                 final URL url = getUrl();
2292                 try {
2293                     if (href.startsWith("http://") || href.startsWith("https://")) {
2294                         baseUrl = new URL(href);
2295                     }
2296                     else if (href.startsWith("//")) {
2297                         baseUrl = new URL(String.format("%s:%s", url.getProtocol(), href));
2298                     }
2299                     else if (href.startsWith("/")) {
2300                         final int port = Window.getPort(url);
2301                         baseUrl = new URL(String.format("%s://%s:%d%s", url.getProtocol(), url.getHost(), port, href));
2302                     }
2303                     else if (url.toString().endsWith("/")) {
2304                         baseUrl = new URL(String.format("%s%s", url.toString(), href));
2305                     }
2306                     else {
2307                         baseUrl = new URL(UrlUtils.resolveUrl(url, href));
2308                     }
2309                 }
2310                 catch (final MalformedURLException e) {
2311                     notifyIncorrectness("Invalid base url: \"" + href + "\", ignoring it");
2312                     baseUrl = url;
2313                 }
2314             }
2315         }
2316 
2317         return baseUrl;
2318     }
2319 
2320     /**
2321      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2322      *
2323      * Adds an {@link AutoCloseable}, which would be closed during the {@link #cleanUp()}.
2324      * @param autoCloseable the autoclosable
2325      */
2326     public void addAutoCloseable(final AutoCloseable autoCloseable) {
2327         if (autoCloseableList_ == null) {
2328             autoCloseableList_ = new ArrayList<>();
2329         }
2330         autoCloseableList_.add(autoCloseable);
2331     }
2332 
2333     /**
2334      * {@inheritDoc}
2335      */
2336     @Override
2337     public boolean handles(final Event event) {
2338         if (Event.TYPE_BLUR.equals(event.getType()) || Event.TYPE_FOCUS.equals(event.getType())) {
2339             return true;
2340         }
2341         return super.handles(event);
2342     }
2343 
2344     /**
2345      * Sets the {@link ElementFromPointHandler}.
2346      * @param elementFromPointHandler the handler
2347      */
2348     public void setElementFromPointHandler(final ElementFromPointHandler elementFromPointHandler) {
2349         elementFromPointHandler_ = elementFromPointHandler;
2350     }
2351 
2352     /**
2353      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2354      *
2355      * Returns the element for the specified x coordinate and the specified y coordinate.
2356      *
2357      * @param x the x offset, in pixels
2358      * @param y the y offset, in pixels
2359      * @return the element for the specified x coordinate and the specified y coordinate
2360      */
2361     public HtmlElement getElementFromPoint(final int x, final int y) {
2362         if (elementFromPointHandler_ == null) {
2363             LOG.warn("ElementFromPointHandler was not specicifed for " + this);
2364             if (x <= 0 || y <= 0) {
2365                 return null;
2366             }
2367             return getBody();
2368         }
2369         return elementFromPointHandler_.getElementFromPoint(this, x, y);
2370     }
2371 
2372     /**
2373      * Moves the focus to the specified element. This will trigger any relevant JavaScript
2374      * event handlers.
2375      *
2376      * @param newElement the element that will receive the focus, use {@code null} to remove focus from any element
2377      * @return true if the specified element now has the focus
2378      * @see #getFocusedElement()
2379      */
2380     public boolean setFocusedElement(final DomElement newElement) {
2381         return setFocusedElement(newElement, false);
2382     }
2383 
2384     /**
2385      * Moves the focus to the specified element. This will trigger any relevant JavaScript
2386      * event handlers.
2387      *
2388      * @param newElement the element that will receive the focus, use {@code null} to remove focus from any element
2389      * @param windowActivated - whether the enclosing window got focus resulting in specified element getting focus
2390      * @return true if the specified element now has the focus
2391      * @see #getFocusedElement()
2392      */
2393     public boolean setFocusedElement(final DomElement newElement, final boolean windowActivated) {
2394         if (elementWithFocus_ == newElement && !windowActivated) {
2395             // nothing to do
2396             return true;
2397         }
2398 
2399         final DomElement oldFocusedElement = elementWithFocus_;
2400         elementWithFocus_ = null;
2401 
2402         if (!windowActivated) {
2403             if (hasFeature(EVENT_FOCUS_IN_FOCUS_OUT_BLUR)) {
2404                 if (oldFocusedElement != null) {
2405                     oldFocusedElement.fireEvent(Event.TYPE_FOCUS_OUT);
2406                 }
2407 
2408                 if (newElement != null) {
2409                     newElement.fireEvent(Event.TYPE_FOCUS_IN);
2410                 }
2411             }
2412 
2413             if (oldFocusedElement != null) {
2414                 oldFocusedElement.removeFocus();
2415                 oldFocusedElement.fireEvent(Event.TYPE_BLUR);
2416             }
2417         }
2418 
2419         elementWithFocus_ = newElement;
2420 
2421         if (elementWithFocus_ instanceof SelectableTextInput
2422                 && hasFeature(PAGE_SELECTION_RANGE_FROM_SELECTABLE_TEXT_INPUT)) {
2423             final SelectableTextInput sti = (SelectableTextInput) elementWithFocus_;
2424             setSelectionRange(new SimpleRange(sti, sti.getSelectionStart(), sti, sti.getSelectionEnd()));
2425         }
2426 
2427         if (elementWithFocus_ != null) {
2428             elementWithFocus_.focus();
2429             elementWithFocus_.fireEvent(Event.TYPE_FOCUS);
2430         }
2431 
2432         if (hasFeature(EVENT_FOCUS_FOCUS_IN_BLUR_OUT)) {
2433             if (oldFocusedElement != null) {
2434                 oldFocusedElement.fireEvent(Event.TYPE_FOCUS_OUT);
2435             }
2436 
2437             if (newElement != null) {
2438                 newElement.fireEvent(Event.TYPE_FOCUS_IN);
2439             }
2440         }
2441 
2442         // If a page reload happened as a result of the focus change then obviously this
2443         // element will not have the focus because its page has gone away.
2444         return this == getEnclosingWindow().getEnclosedPage();
2445     }
2446 
2447     /**
2448      * Returns the element with the focus or null if no element has the focus.
2449      * @return the element with focus or null
2450      * @see #setFocusedElement(DomElement)
2451      */
2452     public DomElement getFocusedElement() {
2453         return elementWithFocus_;
2454     }
2455 
2456     /**
2457      * <p><span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span></p>
2458      *
2459      * Sets the element with focus.
2460      * @param elementWithFocus the element with focus
2461      */
2462     public void setElementWithFocus(final DomElement elementWithFocus) {
2463         elementWithFocus_ = elementWithFocus;
2464     }
2465 
2466     /**
2467      * <p><span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span></p>
2468      *
2469      * <p>Returns the page's current selection ranges. Note that some browsers, like IE, only allow
2470      * a single selection at a time.</p>
2471      *
2472      * @return the page's current selection ranges
2473      */
2474     public List<Range> getSelectionRanges() {
2475         return selectionRanges_;
2476     }
2477 
2478     /**
2479      * <p><span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span></p>
2480      *
2481      * <p>Makes the specified selection range the *only* selection range on this page.</p>
2482      *
2483      * @param selectionRange the selection range
2484      */
2485     public void setSelectionRange(final Range selectionRange) {
2486         selectionRanges_.clear();
2487         selectionRanges_.add(selectionRange);
2488     }
2489 
2490     /**
2491      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2492      *
2493      * Execute a Function in the given context.
2494      *
2495      * @param function the JavaScript Function to call
2496      * @param thisObject the "this" object to be used during invocation
2497      * @param args the arguments to pass into the call
2498      * @param htmlElementScope the HTML element for which this script is being executed
2499      *        This element will be the context during the JavaScript execution. If null,
2500      *        the context will default to the page.
2501      * @return a ScriptResult which will contain both the current page (which may be different than
2502      *        the previous page and a JavaScript result object.
2503      */
2504     public ScriptResult executeJavaScriptFunction(final Object function, final Object thisObject,
2505             final Object[] args, final DomNode htmlElementScope) {
2506         if (!getWebClient().getOptions().isJavaScriptEnabled()) {
2507             return new ScriptResult(null, this);
2508         }
2509 
2510         return executeJavaScriptFunction((Function) function, (Scriptable) thisObject, args, htmlElementScope);
2511     }
2512 
2513     private ScriptResult executeJavaScriptFunction(final Function function, final Scriptable thisObject,
2514             final Object[] args, final DomNode htmlElementScope) {
2515 
2516         final JavaScriptEngine engine = (JavaScriptEngine) getWebClient().getJavaScriptEngine();
2517         final Object result = engine.callFunction(this, function, thisObject, args, htmlElementScope);
2518 
2519         return new ScriptResult(result, getWebClient().getCurrentWindow().getEnclosedPage());
2520     }
2521 
2522     private void writeObject(final ObjectOutputStream oos) throws IOException {
2523         oos.defaultWriteObject();
2524         oos.writeObject(originalCharset_ == null ? null : originalCharset_.name());
2525     }
2526 
2527     private void readObject(final ObjectInputStream ois) throws ClassNotFoundException, IOException {
2528         ois.defaultReadObject();
2529         final String charsetName = (String) ois.readObject();
2530         if (charsetName != null) {
2531             originalCharset_ = Charset.forName(charsetName);
2532         }
2533     }
2534 }