I am using HtmlUnit and trying to get page information. URL I am scraping is: http://www.mpcz.co.in/portal/Bhopal_home.portal?_nfpb=true&_pageLabel=custCentre_viewBill_bpl 'submitButton' click calls a ajax which fails. I am not able to find the root cause of the ajax failure. I debugged it using Visual Debugger.
Input data: IVRS, 23046091095250, Bill Desk Payment
Below is code.
final WebClient webClient = new WebClient(BrowserVersion.FIREFOX_31);
webClient.getOptions().setThrowExceptionOnFailingStatusCode(false);
webClient.getOptions().setThrowExceptionOnScriptError(false);
webClient.getOptions().setCssEnabled(false);
webClient.getOptions().setUseInsecureSSL(true);
CookieHandler
.setDefault(new CookieManager(null, CookiePolicy.ACCEPT_ALL));
webClient.setAjaxController(new NicelyResynchronizingAjaxController());
webClient.getOptions().setTimeout(120000);
webClient.getOptions().setJavaScriptEnabled(true);
webClient.waitForBackgroundJavaScript(120000);
webClient.getOptions().setRedirectEnabled(true);
WebClientUtils.attachVisualDebugger(webClient);
webClient.addWebWindowListener(new WebWindowListener() {
public void webWindowContentChanged(WebWindowEvent event) {
System.out.println("WeBWindow_aaa:"+ event.getEventType());
}
public void webWindowClosed(WebWindowEvent event) {
System.out.println("WeBWindow_aaa:"+ event.getEventType());
}
public void webWindowOpened(WebWindowEvent event) {
System.out.println("WeBWindow_aaa:"+ event.getEventType());
// TODO Auto-generated method stub
}
});
webClient.setJavaScriptErrorListener(new JavaScriptErrorListener() {
@Override
public void timeoutError(HtmlPage page, long arg1, long arg2) {
// TODO Auto-generated method stub
System.out.println("Erroraaa :" + arg1 );
}
@Override
public void scriptException(HtmlPage arg0, ScriptException arg1) {
// TODO Auto-generated method stub
System.out.println("scriptExceptionaaa :" + arg1 );
}
@Override
public void malformedScriptURL(HtmlPage arg0, String arg1,
MalformedURLException arg2) {
// TODO Auto-generated method stub
System.out.println("malformedScriptURLaaa :" + arg1 );
}
@Override
public void loadScriptError(HtmlPage arg0, URL arg1, Exception arg2) {
// TODO Auto-generated method stub
System.out.println("loadScriptErroraaa :" + arg1 );
}
});
HtmlPage page = webClient
.getPage("http://www.mpcz.co.in/portal/Bhopal_home.portal?_nfpb=true&_pageLabel=custCentre_viewBill_bpl");
webClient.waitForBackgroundJavaScript(20000);
System.out.println("Page text: " + page.asText());
HtmlTextInput identifier1Textbox = (HtmlTextInput) page.getElementById("chooseIdentifier1");
identifier1Textbox.setAttribute("value", "IVRS");
HtmlTextInput consumerNoTextbox = (HtmlTextInput) page.getElementById("accntId");
consumerNoTextbox.setAttribute("value", "23046091095250");
HtmlTextInput gatewayTextbox = (HtmlTextInput) page.getElementById("chooseGateway1");
gatewayTextbox.setAttribute("value", "Bill Desk Payment");
HtmlButton submitButton = (HtmlButton) page.getElementById("ext-gen31");
webClient.waitForBackgroundJavaScript(60000);
page = (HtmlPage) submitButton.click();
webClient.waitForBackgroundJavaScript(60000);
System.out.println("Page text: " + page.asText());
System.out.println("Finished");
webClient.close();