I have a class with (currently) 1567 objects. They are urls,title and publication dates of article which I parse from RSS feeds of websites. The cloud job is periodic, so the objects keep on increasing. Although I check for uniqueness in beforesave, from time to time some duplicate items creep up, approximately 10% of objects are duplicate.
I have been trying to make a job to delete these duplicates and want to make a query logic which can get ALL the objects at once. The maximum limit of query is 1000. I referred to this question on Parse Help, and tried to convert it into JavaScript cloud code.
Parse.Cloud.job("DeleteDuplicate", function(request, status) {
var query = new Parse.Query(NewsArticle);
var allObjectArray= [];
var limit = 1000;
var skip = 0;
var repeat = true;
query.limit(limit);
query.skip(skip);
do{
query.find({
success: function(results) {
allObjectArray.push(results.concat());
if(results.length === limit){
skip = skip+limit;
query.skip(skip);
repeat = true;
console.log("true");
}else{
repeat = false;
console.log("false");
}
console.log("Successfully retrieved " + results.length);
},
error: function(error) {
alert("Error: " + error.code + " " + error.message);
status.error("Error: " + error.code + " " + error.message);
}
});
}while(repeat);
status.success("final length "+allObjectArray.length);
});
The code fails with the Job Status of "Could not connect to Cloud Code". I think it goes to infinite loop and times out after 2-3 minutes. If anyone can help, that would be great.
EDIT: Can using Promise help?
EDIT 2: Been trying Promise now -
Parse.Cloud.job("jobFindAll", function(request, status) {
var query = new Parse.Query(NewsArticle);
var allObjectArray= [];
var limit = 1000;
var skip = 0;
var repeat = false;
query.limit(limit);
query.skip(skip);
var promiseList = [];
console.log("in job");
query.find().then(function(results) {
console.log("results.length "+results.length);
allObjectArray = results.slice();
console.log("allObjectArray.length "+allObjectArray.length);
if(results.length === limit){
console.log("smaller");
skip = skip+limit;
do{
console.log("first repeat = "+repeat);
promiseList.push(functionFindAll(allObjectArray, limit, skip));
console.log("promiseList - "+promiseList);
repeat = promiseList[promiseList.length-1];
console.log("looping repeat = "+repeat);
}while(repeat);
return Parse.Promise.when(promiseList);
}else{
console.log("longer");
}
}).then(function(){
console.log("in then");
status.success("final length "+allObjectArray.length);
}, function(error) {
status.error("Error: " + error.code + " " + error.message);
}
);
});
function functionFindAll(allObjectArray, limit, skip){
var returnPromiseList = [];
var query_new = new Parse.Query(NewsArticle);
query_new.limit(limit);
query_new.skip(skip);
query_new.find().then(function(results) {
console.log("function results.length "+results.length);
if(results.length === limit){
skip = skip+limit;
query.skip(skip);
allObjectArray.push(results.concat());
console.log("true in function");
return Parse.Promise.as(true);
}else{
allObjectArray.push(results.concat());
return Parse.Promise.as(false);
console.log("false in function");
}
},
function(error) {
console.log("Error: " + error.code + " " + error.message);
return Parse.Promise.as("ERROR!");
}
);
console.log("below "+allObjectArray.length);
}
Now the code doesnt enter the query_new.find(). The log messages in the function doesnt appear, directly the message "below...." appears.
The following code will find all the items in the class, it does not use skip because parse.com has another freaking limit call "unable to skip more than 10000 items". it skip using the objectId.
Parse.Cloud.job("findAll", function(request, status) {
var result = [];
var processCallback = function(res) {
result = result.concat(res);
if (res.length === 1000) {
process(res[res.length-1].id);
return;
}
// do something about the result, result is all the object you needed.
status.success("final length " + result.length);
}
var process = function(skip) {
var query = new Parse.Query("NewsArticle");
if (skip) {
console.log("in if");
query.greaterThan("objectId", skip);
}
query.limit(1000);
query.ascending("objectId");
query.find().then(function querySuccess(res) {
processCallback(res);
}, function queryFailed(reason) {
status.error("query unsuccessful, length of result " + result.length + ", error:" + error.code + " " + error.message);
});
}
process(false);
});
All JS calls in Parse are asynchronous, so your function is spawning infinite threads (even if it was synchronous, your query.skip(skip) would need to go inside the while loop not before it.
Try moving the query out to its own function, which is called recursively on success until you have read all of your objects. You should be able to adapt the code in this answer, which is for a similar problem: https://stackoverflow.com/a/17268263/1176247
Related
I am developing an office 365 word web addin, wherein I need to upload the currently opened document to my server. For which I am trying to get the file data using following code.
The method getSliceAsync() is returning only first slice of data.
On debugging it gives "Addin Error: Sorry, we had to restart because this addin wasn't responding" while getting second slice.
I am using this link for reference : [https://learn.microsoft.com/en-us/office/dev/add-ins/word/get-the-whole-document-from-an-add-in-for-word][1]
Here is my code:
Office.context.document.getFileAsync(Office.FileType.Compressed, { sliceSize: 65536 }, function (result) {
if (result.status == "succeeded") {
// If the getFileAsync call succeeded, then result.value will return a valid File Object
var myFile = result.value;
var filename1 = myFile.name;
console.log(filename1);
var sliceCount = myFile.sliceCount;
var slicesReceived = 0, isAllSlicesSuccess = true, docdataSlices = [];
// document.getElementById("result").innerText = "File size:" + myFile.size + "#Slices: " + sliceCount;
console.log(" File size:" + myFile.size + " #Slices: " + sliceCount, "");
makeProgress(20);
// Iterate over the file slices
for (var i = 0; i < sliceCount && isAllSlicesSuccess; i++) {
var diffPercent = ((i / sliceCount) * 100);
myFile.getSliceAsync(i, function (sliceResult) {
if (sliceResult.status == "succeeded") {
if (!isAllSlicesSuccess) { // Some slice has failed to get, no need to continue
console.log("Error", "One slice failed to get");
return;
console.log(sliceResult);
}
console.log('sliceResult', sliceResult);
console.log("Success", "i: " + i);
console.log("++slicesReceived ",slicesReceived );
console.log(" sliceCount",sliceCount );
console.log("++slicesReceived == sliceCount",slicesReceived == sliceCount);
// One chunk was got, store it in a temporal array
// ++slicesReceived;
// or you can do something with the chunk, such as sent it to a third party server
docdataSlices[sliceResult.value.index] = sliceResult.value.data;
if (++slicesReceived == sliceCount) {
getAllSlicesTime = Date.now();
var performance = (getAllSlicesTime - startTime) / 1000.0;
console.log("Success", "All slices has been get, Seconds: " + performance);
// All slices have been received
myFile.closeAsync(function (closeRes) {
if (closeRes.status == "succeeded") {
console.log("Close Success", "Success");
// DUClick();
}
else {
console.log("Close Error", closeRes.error.message);
}
});
onGetAllSlicesSucceeded(docdataSlices, false);
}
}
else {
isAllSlicesSuccess = false;
myFile.closeAsync(function (closeRes) {
if (closeRes.status == "succeeded") {
console.log("Close Success", "Success");
// DUClick();
}
else {
console.log("Close Error", closeRes.error.message);
}
});
console.log("Get Slice Error:", sliceResult.error.message);
}
});
}
}
else {
getFileTime = Date.now();
var performance = (getFileTime - startTime) / 1000.0;
console.log('Get File Error:', "Seconds: " + performance + " " + result.error.message);
}
});
Please suggest! Thanks in advance!
[1]: https://learn.microsoft.com/en-us/office/dev/add-ins/word/get-the-whole-document-from-an-add-in-for-word
I am requesting a rest server from nodejs by using nodejs request module.
I want to cancel stream if incoming data size is out of allowed limit.the purpose here is to ensure that my network is not locked.
My code example is as follows;
var http = require("http");
async function httpRequest({
host,
method,
port,
path
} = params, data) {
if (method.toUpperCase() === "GET") {
let query = "";
data = JSON.parse(data);
for (var key in data) {
if (data.hasOwnProperty(key)) {
let value = data[key];
console.log(key + " -> " + value);
query = query
.concat("&")
.concat(key)
.concat("=")
.concat(value);
}
}
if (query) {
query = "?".concat(query.substring(1));
}
path = encodeURI(path.concat(query));
console.log("path : " + path);
}
var opts = {
hostname: host,
port: port,
path: path,
method: method,
timeout: 30 * 1000,
headers: {
"Content-Type": "application/json"
}
};
return new Promise(function (resolve, reject) {
const req = http.request(opts, function (response) {
console.log("Status Code : " + response.statusCode);
if (response.statusCode < 200 || response.statusCode >= 300) {
req.end();
return reject("Fetch data failed = " + response.statusCode);
}
var str = "";
response.on("data", function (chunk) {
console.log("chunk : " + chunk);
str += chunk;
if (str.length > 256) {
req.abort();
reject(
new Error(
"The size of the incoming data is larger than the allowable limit."
)
);
}
});
response.on("end", function () {
console.log("\n Result from web service : ", str);
try {
let jsonData = JSON.parse(str);
if (jsonData.status) {
if (jsonData.status.toLowerCase === "success") {
if (!("result" in jsonData)) {
reject("Json Structure Error");
}
} else if (jsonData.status.toLowerCase === "error") {
if (!jsonData.error) {
reject("Json Structure Error");
}
}
resolve(jsonData);
} else {
reject("Json Structure Error");
}
} catch (error) {
reject("Response json error : " + error);
}
});
});
if (method.toUpperCase() !== "GET" && data) {
req.write(data);
}
//req bitti
req.on("timeout", function () {
console.log("timeout! " + opts.timeout / 1000 + " seconds expired");
req.abort();
});
req.on("error", function (err) {
console.log("Error : " + err);
if (err.code === "ECONNRESET") {
req.abort();
console.log("Timeout occurs : " + err);
reject(new Error("Timeout occurs : " + err));
} else if (err.code === "ENOTFOUND") {
req.abort();
console.log("Address cannot be reachable : " + err);
reject(new Error("Address cannot be reachable : " + err));
} else {
req.abort();
reject(new Error(err));
}
});
req.end();
});
}
let data = JSON.stringify({
username: "monetrum",
password: "123456",
name: "Loremipsumdolorsitamet,consecteturadipiscingelit" +
".Aeneaninaliquamodio,egetfac"
});
let params = {
host: "127.0.0.1",
method: "GET",
port: 3010,
path: "/login"
};
httpRequest(params, data);
So farr so good.But There is a problem.I am controlling incoming data.Size of data I allowed must not greater than 256 Bytes.But first fetch of chunk is larger than allowed size.So my size control is nonsense.Is there a way to handle it.Is it possible to limit size of chunk. Thanks in advance.
The 'readable' event
You want to use the readable event instead of the data event:
var byteCount = 0;
var chunkSize = 32;
var maxBytes = 256;
req.on('readable', function () {
var chunks = [];
var data;
while(true) {
data = this.read(chunkSize);
if (!data) { break; }
byteCount += data.byteLength;
if (byteCount > maxBytes) {
req.abort();
break;
}
chunks.push(data);
}
// do something with chunks
});
req.on('abort', function () {
// do something to handle the error
});
Since your question is very specific I made the example a little more generic so that hopefully others will be able to glean from it as well.
See https://nodejs.org/api/stream.html#stream_event_readable
However...
The Network Does't Care
However, you're going to get more data than that. TCP packet size is 64k. Over non-gigabit ethernet that gets MTU truncated to 1500 bytes (1.5k).
There's nothing that you can do to prevent the network activity from happening other than closing the connection, and you can't get less than 1.5k of data per data event unless there is less than 1.5k of data being sent (or crazy network issues happen, which you have no control over).t
P.S.
I'd recommend that you use a code editor, like VSCode. It's very difficult to read code that has mixes of tabs and spaces and different blocks at different levels. It will suggest plugins that can help you catch mistakes earlier and reformat your code so that it's easier for others (and yourself) to read it.
https://code.visualstudio.com/
https://code.visualstudio.com/docs/languages/javascript
https://marketplace.visualstudio.com/items?itemName=esbenp.prettier-vscode
Several attempts which I tried to add { or ) on missing input. I've asked different question as I missed commas and brackets and also indenting wrong way. I couldn't find what is the missing { or ) in this code.
Parse.Cloud.afterSave("StatusUpdate", function(request) {
if(request.object.existed()) {
return;
}
// if the user is Undefined
var toUser = request.object.get("user") ---< **** Line 8 ****
if (!toUser)
{
throw "Undefined toUser. Skipping push for Activity " + request.object.get('StatusUpdate') + " : " + request.object.id;
return;
}
// put Installation query here if needed
// send notifications based on the StatusUpdate and the passed in user from iOS ---< **** Line 16 ****
Parse.Cloud.useMasterKey() // depreciated in self hosted parse servers
// create query based on passed in values
Parse.Cloud.define("StatusUpdate", function(request, response) {
//query Installation for user
var Installationquery = Parse.Object.extend("Installation");
var query = new Parse.Query(Installationquery);
var message = request.params.message
query.equalTo("user", request.params.User);
query.find({
success: function(results) {
response.success("found user" + results)
// Do something with the returned Parse.Object values
for (var i = 0; i < results.length; i++) {
var object = results[i];
Parse.Push.send({
where: query, // Set our Installation query
data: {
alert: createMessage(message),
badge: "Increment",
sound: ""
}
}, {
success: function() {
// Push was successful
console.log("sent ")
},
error: function(error) {
console.log("Error " + error)
}
});
}
},
error: function(error) {
alert("Error: " + error.code + " " + error.message);
}
});
});
//})
var alertMessage = function createMessage(request) {
var message = "";
if (request.object.get("StatusUpdate") === "likedby") {
if (request.user.get('postedby')) {
message = request.user.get('postedby') + ': ' + request.object.get('statusOBJID').trim();
} else {
message = "Someone liked on your status update.";
}
// Trim our message to 140 characters.
if (message.length > 140) {
message = message.substring(0, 140);
}
return message;
}
}
the problems with your code:
Line 1: Unmatched '{'.
Line 7: Missing semicolon.
Line 18: Missing semicolon.
Line 26: Missing semicolon.
Line 30: Missing semicolon.
Line 45: Missing semicolon.
Line 48: Missing semicolon.
Line 76: Missing semicolon.
Line 76: Expected ')' and instead saw ''.
Line 76: Missing semicolon.
fixed:
Parse.Cloud.afterSave("StatusUpdate", function(request) {
if(request.object.existed()) {
return;
}
});
// if the user is Undefined
var toUser = request.object.get("user");
if (!toUser)
{
throw "Undefined toUser. Skipping push for Activity " + request.object.get('StatusUpdate') + " : " + request.object.id;
return;
}
// put Installation query here if needed
// send notifications based on the StatusUpdate and the passed in user from ios
Parse.Cloud.useMasterKey(); // depreciated in self hosted parse servers
// create query based on passed in values
Parse.Cloud.define("StatusUpdate", function(request, response) {
//query Installation for user
var Installationquery = Parse.Object.extend("Installation");
var query = new Parse.Query(Installationquery);
var message = request.params.message;
query.equalTo("user", request.params.User);
query.find({
success: function(results) {
response.success("found user" + results);
// Do something with the returned Parse.Object values
for (var i = 0; i < results.length; i++) {
var object = results[i];
Parse.Push.send({
where: query, // Set our Installation query
data: {
alert: createMessage(message),
badge: "Increment",
sound: ""
}
}, {
success: function() {
// Push was successful
console.log("sent ");
},
error: function(error) {
console.log("Error " + error);
}
});
}
},
error: function(error) {
alert("Error: " + error.code + " " + error.message);
}
});
});
//})
var alertMessage = function createMessage(request) {
var message = "";
if (request.object.get("StatusUpdate") === "likedby") {
if (request.user.get('postedby')) {
message = request.user.get('postedby') + ': ' + request.object.get('statusOBJID').trim();
} else {
message = "Someone liked on your status update.";
}
// Trim our message to 140 characters.
if (message.length > 140) {
message = message.substring(0, 140);
}
return message;
}
};
I'm using Node to run this program, where is my error(s)? It's saying I'm missing ) after argument list. I can't find where this error is, I've tried putting the ) in various places. I'm using Node v5
var Twit = require('twit');
var T = new Twit(require('./config.js'));
var stream = T.stream('statuses/filter', {
track: 'xoxo, oi, i\m fine,'
});
(stream.on('tweet', function(tweet) {
console.log('#' + tweet.user.screen_name + ': ' + tweet.text);
if (tweet.text.indexOf('RT') > -1) {
return;
}
var replyString;
if (tweet.user.utc_offset === null) {
replyString = ' Ok';
} else {
replyString = ' Okay';
}
})
(T.post('statuses/update', {
status: '#' + tweet.user.screen_name + replyString,
in_reply_to_status_id: tweet.id_str
}, function(err, data, response) {
if (err) {
console.log(err);
return;
}
}
tweet.botReplyId = data.id_str);
db.tweets.insert(tweet);
});
(end)
})
setInterval(stream, 60000);
The code seems to be a bit all over the place with regards to scope and it makes it a bit difficult to follow.
Try using something like the following which annotates it a bit and should help avoid issues like this (as it seems to validate without any errors) :
// Define your variables
var Twit = require('twit');
var T = new Twit(require('./config.js'));
var stream = T.stream('statuses/filter', { track: 'xoxo, oi, i\'m fine,'});
// When a tweet occurs
(stream.on('tweet', function(tweet) {
// Log it
console.log('#' + tweet.user.screen_name + ': ' + tweet.text);
// Determine if it is a retweet and ignore
if (tweet.text.indexOf('RT') > -1) { return; }
// Set your reply
var replyString = (tweet.user.utc_offset === null) ? ' Ok' : ' Okay';
// Post your reply
T.post('statuses/update', { status: '#' + tweet.user.screen_name + replyString, in_reply_to_status_id: tweet.id_str}, function(err, data, response) {
// If an error occurs, log it
if (err) {
console.log(err);
return;
}
// Otherwise store your response and store it
tweet.botReplyId = data.id_str;
db.tweets.insert(tweet);
});
}));
// Check your stream every 10 minutes
setInterval(stream, 60000);
I'm trying to create a cloud Job that takes the users full name or username and saves it in lower case in another column. here's what I have so far:
Parse.Cloud.job('normaliseUsername',function(request, status) {
Parse.Cloud.useMasterKey();
var query = new Parse.Query(Parse.User);
query.find({
success: function(items){
for (var i=0;i<items.length;i++) {
var user = items[i];
console.log(user);
var changed = user["nameChanged"];
if (changed === true) {
var username = user.username;
user.set("lowerCaseName",username.toLowerCase());
} else {
var realName = user["firstName"] + " " + user["lastName"];
user.set("lowerCaseName",realName.toLowerCase());
}
user.save();
}
}
});
});
This results in a new column, lowerCaseName, full of undefined.
how do I access properties of a PFUser in this instance? I have tried using user.get(''); but it says Cannot call method 'get' of undefined
Do it this way :
Parse.Cloud.job("normaliseUsername", function(request, status) {
Parse.Cloud.useMasterKey();
var count_user = 0;
var query = new Parse.Query(Parse.User);
query.descending('updatedAt');
query.Exist('nameChanged');
query.limit(1000);
return query.find().then(function(users) {
return Parse.Promise.when(users.map(function(user) {
count_user+= 1;
if (user.get("nameChanged")) {
user.set("canonical_firstname", user.get("username").toLowerCase());
} else {
var realName = user.get("firstname") + ' ' + user.get("lastname");
user.set("lowerCaseName", realName.toLowerCase());
}
return user.save();
}));
}).then(function() {
status.success("normaliseUsername with " + count_user + " user(s) updated.");
}, function(error) {
status.error("Uh oh, something went wrong.");
});
});
Your loop with for, will never works, you need to use Promise. More information here : http://blog.parse.com/learn/engineering/whats-so-great-about-javascript-promises/
The way the above script works, you will work with Promise in Parallel, not in Series : https://parse.com/docs/js/guide#promises