I've got a problem with my web-scraper. It should execute a javascript to make some changes on a webpage, then download the source and extract some specific data.
bool working = true;
for(int i = 0; i < urls.Count(); ++i)
{
working = true;
ChromiumWebBrowser scraper = new ChromiumWebBrowser(urls[i]);
scraper.FrameLoadEnd += scrape;
while(working)
{
Application.DoEvents();
System.Threading.Thread.Sleep(50);
}
}
async void scrape(object sender, FrameLoadEndEventArgs args)
{
ChromiumWebBrowser chrome = (ChromiumWebBrowser)sender;
if (args.Frame.IsMain && chrome.CanExecuteJavascriptInMainFrame)
{
string script = Properties.Resources.script;
chrome.ExecuteScriptAsync(script);
string data = " ";
do
{
string html = await chrome.GetSourceAsync();
string dataField = "data";
int dataFieldIndex = html.IndexOf(phoneField);
data = html.Substring(dataFieldIndex + dataField.Count(), html.IndexOf("<", dataFieldIndex + dataField.Count()) - dataFieldIndex - dataField.Count());
System.Threading.Thread.Sleep(50);
} while (data.Count() == 3);
addDataToHashSet(data);
}
else
{
Debug.WriteLine("Error");
}
}
private void addDataToHashSet(string data)
{
data = data.Replace("-", "");
data = data.Replace(" ", "");
dataHashSet.Add(data);
working = false;
}
Unfortunately, it's unpredictable. It hangs after few iterations - a proper event with CanExecuteJavascriptInMainFrame and IsMain is never triggered for some iterations. Could you explain it to me?
[edit]: My Cef initialization below:
CefSettings cefSettings = new CefSettings();
cefSettings.CefCommandLineArgs.Add("allow-running-insecure-content", "1");
cefSettings.IgnoreCertificateErrors = true;
CefSharpSettings.Proxy = new ProxyOptions(ip: "...", port: "...");
Cef.Initialize(cefSettings);
Related
Created Office Add-ins (App ID 42949681619)
Implemented enter link description here.
var decodedToken = getUrlVars()["et"];
var decodedBytes = encodeURIComponent(decodedToken);
var Valid = false;
function getUrlVars() {
var vars = [], hash;
var hashes = window.location.href.slice(window.location.href.indexOf('?') + 1).split('&');
for (var i = 0; i < hashes.length; i++) {
hash = hashes[i].split('=');
vars.push(hash[0]);
vars[hash[0]] = hash[1];
}
return vars;
}
function EtokenWeb(sourceWord) {
try {
var xhr = new XMLHttpRequest();
var translateRequest = "../Verification.asmx/VerifyEntitlement?et="+ sourceWord;
xhr.open("POST", translateRequest);
xhr.responseType = 'document';
xhr.onload = function () {
var result = parseResponse(xhr.responseXML);
Valid = result;
}
// Send the HTTP request.
xhr.send();
}
catch (ex) {
app.showNotification(ex.name, ex.message);
}
}
function parseResponse(responseText) {
var response = "Cannot read response.",
xmlDoc;
try {
response = responseText.getElementsByTagName("string")[0].firstChild.nodeValue;
}
catch (ex) {
app.showNotification(ex.name, ex.message);
}
return response;
}
using BenWeb.VerificationService;
using System;
using System.Collections.Generic;
using System.Collections.Specialized;
using System.Linq;
using System.Web;
using System.Web.Services;
namespace BenWeb
{
[WebService(Namespace = "http://tempuri.org/")]
[WebServiceBinding(ConformsTo = WsiProfiles.BasicProfile1_1)]
[System.ComponentModel.ToolboxItem(false)]
[System.Web.Script.Services.ScriptService]
public class Verification : System.Web.Services.WebService
{
private static VerificationServiceClient service = new VerificationServiceClient();
VerifyEntitlementTokenRequest request = new VerifyEntitlementTokenRequest();
[WebMethod]
public bool VerifyEntitlement ()
{
// Store the URL from the incoming request and declare
// strings to store the query and the translation result.
string currentURL = HttpContext.Current.Request.RawUrl;
string queryString;
bool result = false;
try
{
// Check to make sure some query string variables exist.
int indexQueryString = currentURL.IndexOf('?');
if (indexQueryString >= 0)
{
queryString = (indexQueryString < currentURL.Length - 1) ? currentURL.Substring(indexQueryString + 1) : String.Empty;
// Parse the query string variables into a NameValueCollection.
NameValueCollection queryStringCollection = HttpUtility.ParseQueryString(queryString);
string Token = queryStringCollection["et"];
if (Token != null)
{
request.EntitlementToken = Token;
VerifyEntitlementTokenResponse omexResponse = service.VerifyEntitlementToken(request);
result = omexResponse.IsValid;
}
else result = false;
}
}
catch (Exception ex)
{
//result = ex.Message;
result = false;
}
// to the HTTP request.
return result;
}
}
}
Added in code:
if (Valid)
{
$('#highlight-button').click(SelectedRange);
$('#highlight2-button').click(Recalculation);
}
Sent for approval. Received a comment "Your add-in does not work as described in your addin description. Nothing happens when we click the “Choose” button."
Test license VerificationService worked fine (IsTest=True, IsValid=False).
Where I went wrong?
And how to check for this (correct) license?
*This is a school project
I'v been using this project on couple of computers, in the first computer my project run perfectly, but on other computer it's giving me a System.AccessViolationException on 'random' places on code.
I'm using web service, asmx page and aspx pages, my database is Microsoft Access.
I'm trying to belive the bug is a problem with my second computer settings and not on my code, because the code run perfect on others computer.
This is one place that giving me a bug:
javascript:
function sendList(r, id, subject, content) {
$('#modal1').modal('close');
$.getJSON("/API/UIService.asmx/SendMessage?currentUserID=" + id + "&reciverID=" + r + "&subject=" + subject + "&content=" + content, function (msg) {
msg = fixMsg(msg);
if (msg == "Message sent") {
modalAlert(fixMsg(msg));
inbox.update();
sent.update();
}
else
alert(msg);
});
}
Web service method:
[WebMethod]
public void SendMessage(int currentUserId, string reciverID, string subject, string content)
{
try
{
string[] res = reciverID.Split(',');
List<Message> m = new List<Message>();
for (int i = 0; i < res.Length; i++)
{
Message msg = new Message()
{
SenderID = currentUserId,
ReciverID = int.Parse(res[i]),
Subject = subject.Replace("'", ""),
Content = content.Replace("'", ""),
IsRead = false,
SendDate = DateTime.Now.ToLocalTime(),
DeletedBy = -1
};
m.Add(msg);
}
MessageService.SendList(m);
var jsonSerialiser = new JavaScriptSerializer();
Context.Response.Write(jsonSerialiser.Serialize("Message sent"));
}
catch (Exception ex)
{
var jsonSerialiser = new JavaScriptSerializer();
Context.Response.Write(jsonSerialiser.Serialize("Server Error: " + ex.Message));
}
}
MessageService.SendList method:
public static void SendList(List<Message> msgs)
{
DataTable messages = new DataTable();
messages.Columns.Add("MessageSenderID");
messages.Columns.Add("MessageReciverID");
messages.Columns.Add("MessageSubject");
messages.Columns.Add("MessageContent");
messages.Columns.Add("MessageIsRead");
messages.Columns.Add("MessageSendDate");
messages.Columns.Add("MessageDeletedBy");
for (int i = 0; i < msgs.Count; i++)
{
DataRow dr = messages.NewRow();
dr["MessageSenderID"] = msgs[i].SenderID;
dr["MessageReciverID"] = msgs[i].ReciverID;
dr["MessageSubject"] = msgs[i].Subject;
dr["MessageContent"] = msgs[i].Content;
dr["MessageIsRead"] = false;
dr["MessageSendDate"] = DateTime.Now;
dr["MessageDeletedBy"] = -1;
messages.Rows.Add(dr);
}
OleDbConnection con = new OleDbConnection(Connect.GetConnectionString());
OleDbCommand cmd = new OleDbCommand();
cmd.Connection = con;
cmd.CommandText = "INSERT INTO Messages(MessageSenderID, MessageReciverID, MessageSubject, MessageContent, " +
"MessageIsRead, MessageSendDate, MessageDeletedBy) " +
"VALUES " +
"(#MessageSenderID, #MessageReciverID, #MessageSubject, #MessageContent, #MessageIsRead, " +
"#MessageSendDate, #MessageDeletedBy)";
cmd.Parameters.Add("#MessageSenderID", OleDbType.Integer);
cmd.Parameters.Add("#MessageReciverID", OleDbType.Integer);
cmd.Parameters.Add("#MessageSubject", OleDbType.WChar, 255);
cmd.Parameters.Add("#MessageContent", OleDbType.WChar, 255);
cmd.Parameters.Add("#MessageIsRead", OleDbType.Boolean, 2);
cmd.Parameters.Add("#MessageSendDate", OleDbType.Date);
cmd.Parameters.Add("#MessageDeletedBy", OleDbType.Integer);
con.Open();
cmd.Prepare();
for (int i = 0; i < messages.Rows.Count; i++)
{
cmd.Parameters[0].Value = int.Parse(messages.Rows[i]["MessageSenderID"].ToString());
cmd.Parameters[1].Value = int.Parse(messages.Rows[i]["MessageReciverID"].ToString());
cmd.Parameters[2].Value = messages.Rows[i]["MessageSubject"].ToString();
cmd.Parameters[3].Value = messages.Rows[i]["MessageContent"].ToString();
cmd.Parameters[4].Value = bool.Parse(messages.Rows[i]["MessageIsRead"].ToString());
cmd.Parameters[5].Value = DateTime.Parse(messages.Rows[i]["MessageSendDate"].ToString());
cmd.Parameters[6].Value = int.Parse(messages.Rows[i]["MessageDeletedBy"].ToString());
cmd.ExecuteNonQuery();
}
OleDbTransaction trance = con.BeginTransaction();
cmd.Transaction = trance;
trance.Commit();
con.Close();
}
The code stuck at con.Open(); and giving: System.AccessViolationException Attempted to read or write protected memory. This is often an indication that other memory has been corrupted
Here is my simple edit box control with typeAhead enabled:
<xp:inputText id="inputNameEditBox">
<xp:typeAhead mode="full" minChars="3" ignoreCase="true"
valueList="#{javascript:return typeAheadList();}"
var="searchValue" valueMarkup="true" id="typeAhead1">
</xp:typeAhead>
</xp:inputText>
And here is SSJS founction call for typeAhead:
var v=new typeAheadTools.personLookup();
function typeAheadList(){
var personsList = v.getPersonsList(searchValue);
var returnList = "<ul>";
if(personsList.length>0){
for (var i=0; i<personsList.length; i++) {
returnList += ["<li>",personsList[i],"</li>"].join("");
}
} else {
returnList += ["<li>","None found","</li>"].join("");
}
returnList += "</ul>";
return returnList;
}
The problem is that I pull list of users by using external Java API and it's very slow:
package typeAheadTools;
public class personLookup {
public ArrayList<String> getPersonsList(String searchString) throws IOException {
URL url = null;
BufferedReader in = null;
HttpURLConnection connection = null;
ArrayList<String> personsList = new ArrayList<String>();
try {
url = new URL("http://aaaa.bbbb.com/DirectoryAPI/abc?person=" + searchString);
connection = (HttpURLConnection) url.openConnection();
connection.setRequestMethod("GET");
connection.setReadTimeout(300 * 1000);
connection.connect();
in = new BufferedReader(new InputStreamReader(connection.getInputStream()));
String inputLine;
while ((inputLine = in.readLine()) != null){
personsList.add(inputLine.trim());
}
in.close();
url = null;
return personsList;
} catch (Exception e) {
e.printStackTrace();
return personsList;
} finally {
url = null;
if (in != null) {
in.close();
in = null;
}
if (connection != null) {
connection.disconnect();
connection = null;
}
}
}
}
The API may return either plain text or JSON. But the key question - is there any other approach I can parse that URL output for typeAhead function. Is REST Service the right control to use instead of Java bean?
When I run this code repeated - except the first time, null pointer exception is thrown even though the file already exists.
static public class writeexcel {
public void write(){
try{
File f = new File("C:/Users/acer/Desktop/2.xls");
Workbook existingbook=null;
WritableWorkbook ww;
WritableSheet sr;
if(!f.exists()){
ww = Workbook.createWorkbook(f);
sr = ww.createSheet("mysheet", 0);
}else{
existingbook = Workbook.getWorkbook(f);
ww = Workbook.createWorkbook(f, existingbook);
sr = ww.getSheet("Mysheet");
}
Label lb = new Label(0, 0, " success");
Label lb1 = new Label (1,5,"new task");
sr.addCell(lb);
sr.addCell(lb1);
ww.write();
ww.close();
}catch(Exception e){
e.printStackTrace();
}
}
}
public static void main(String[] args) {
// TODO Auto-generated method stub
writeexcel exl = new writeexcel() ;
exl.write();
}
}
Try beloe code, it will write "Success"in 100 cells and rows. you can then modify it as per your requirement.
public class Test {
public static void main(final String[] args) {
Test exl = new Test();
exl.write();
}
public void write() {
try {
File f = new File("fileLocation.xls");
Workbook existingbook = null;
WritableWorkbook ww;
WritableSheet sr;
if (!f.exists()) {
ww = Workbook.createWorkbook(f);
sr = ww.createSheet("mysheet", 0);
} else {
existingbook = Workbook.getWorkbook(f);
ww = Workbook.createWorkbook(f, existingbook);
sr = ww.getSheet("Mysheet");
}
for (int i = 0; i < 100; i++) {
for (int x = 0; x < 100; x++) {
Label lb1 = new Label(i + 1, x + 1, "new task");
// Label lb = new Label(i, x, " success");
// sr.addCell(lb);
sr.addCell(lb1);
}
}
ww.write();
ww.close();
} catch (Exception e) {
e.printStackTrace();
}
}
}
I have loaded a web page in BB as follow
//RegBrowserFieldConfig extends BrowserFieldConfig
RegBrowserFieldConfig regBrowserFieldConfig = new RegBrowserFieldConfig();
//RegBrowserFieldListener extends BrowserFieldListener
RegBrowserFieldListener regBrowserFieldListener = new RegBrowserFieldListener();
BrowserField registrationBrowserField = new BrowserField(regBrowserFieldConfig);
registrationBrowserField.addListener(regBrowserFieldListener);
add(registrationBrowserField);
registrationBrowserField.requestContent("http://myurl.com/");
That web page loads fine. There is a submit button in that web page which call onsubmit in the form element in HTML. That is calling to a JavaScript function. With in that function there are some other URL that will fire according to the requirements.
What I need is to get the response of those URL calls. How can I do that?
I tried this way..
BrowserFieldListener list = new BrowserFieldListener() {
public void documentLoaded(BrowserField browserField,
Document document) throws Exception {
String url = document.getBaseURI(); //u can get the current url here... u can use ur logic to get the url after clicking the submit button
Serverconnection(url);//from this methode u can get the response
}
};
browserField.addListener(list);
Serverconnection..
public String Serverconnection(String url) {
String line = "";
// if (DeviceInfo.isSimulator()) {
// url = url + ";deviceSide=true";
// } else {
// url = url + ";deviceSide=true";
// }
url = url + getConnectionString();
try {
HttpConnection s = (HttpConnection) Connector.open(url);
s.setRequestProperty("Content-Type",
"application/x-www-form-urlencoded");
s.setRequestProperty(
"Accept",
"text/html,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5");
s.setRequestProperty(HttpProtocolConstants.HEADER_ACCEPT_CHARSET,
"UTF-8");
s.setRequestMethod(HttpConnection.GET);
InputStream input = s.openInputStream();
byte[] data = new byte[10240];
int len = 0;
StringBuffer raw = new StringBuffer();
while (-1 != (len = input.read(data))) {
raw.append(new String(data, 0, len));
}
line = raw.toString();
input.close();
s.close();
} catch (Exception e) {
System.out.println("response--- excep" + line + e.getMessage());
}
return line;
}
EDIT..
private static String getConnectionString() {
String connectionString = "";
if (WLANInfo.getWLANState() == WLANInfo.WLAN_STATE_CONNECTED) {
connectionString = "?;interface=wifi";
}
else if ((CoverageInfo.getCoverageStatus() & CoverageInfo.COVERAGE_MDS) == CoverageInfo.COVERAGE_MDS) {
connectionString = "?;&deviceside=false";
} else if ((CoverageInfo.getCoverageStatus() & CoverageInfo.COVERAGE_DIRECT) == CoverageInfo.COVERAGE_DIRECT) {
String carrierUid = getCarrierBIBSUid();
if (carrierUid == null) {
connectionString = "?;deviceside=true";
} else {
connectionString = "?;deviceside=false?;connectionUID="
+ carrierUid + "?;ConnectionType=mds-public";
}
} else if (CoverageInfo.getCoverageStatus() == CoverageInfo.COVERAGE_NONE) {
}
return connectionString;
}
private static String getCarrierBIBSUid() {
ServiceRecord[] records = ServiceBook.getSB().getRecords();
int currentRecord;
for (currentRecord = 0; currentRecord < records.length; currentRecord++) {
if (records[currentRecord].getCid().toLowerCase().equals("ippp")) {
if (records[currentRecord].getName().toLowerCase()
.indexOf("bibs") >= 0) {
return records[currentRecord].getUid();
}
}
}
return null;
}