I am making an add on that processes content from a Google Sheets doc and returns it (for now) to new sheet in the spreadsheet. The first iteration works but is deadly slow. I think it's because it processes the data sheet one line at a time. So I'm rewriting it to ingest all the data and then process the array it lives in.
Problem is the amount of data appears to be prohibitive. I can see this in Logger output, in execution transcript, and in stackdriver logging; it gets to about line 35 (11 columns) before it runs out of room.
here is the original version (takes an insane amount of time):
'''
var compareDate = selDate;
var bulletin = ss.getSheetByName(reportSheetName);
var responses = ss.getSheetByName("Calendar");
var lastRow = responses.getLastRow(); //get last row of data sheet
var lastBull = bulletin.getLastRow(); //get last row of bulletin sheet
var nextBullRow = lastBull+1;
var nextBullItem = bulletin.getRange(nextBullRow,1);
for(var i = 2; i <= lastRow; i++) {
var row = responses.getRange(i, 1, 1, 11).getValues();
var dateA = new Date((row[0][4]).valueOf());
var eventType = row[0][0].valueOf();
var eventCalDate = new Date(row[0][4].valueOf());
var eventEvent = row[0][3].valueOf();
var eventOpp = row[0][7].valueOf();
var eventLocation = row[0][8].valueOf();
var eventMonth = eventCalDate.getMonth();
var eventDate = eventCalDate.getDate();
var eventYear = eventCalDate.getYear();
var eventShortDate = eventMonth+"/"+eventDate+"/"+eventYear;
var start = new Date(thisWeekSt);
var end = new Date(thisWeekEnd);
/*var masterDate = new Date(compareDate).valueOf();
var thisCat = row[0][3];
var bullItem = row[0][4]; */
var dateMatch = false;
if(dateA > start && dateA < end) {
dateMatch=true;
bulletin.getRange(nextBullRow,1,nextBullRow,6).setFontWeight("normal");
bulletin.getRange(nextBullRow,typeCol).setValue(eventType);
bulletin.getRange(nextBullRow,dateCol).setValue(eventShortDate);
bulletin.getRange(nextBullRow,timeCol).setValue(eventType);
bulletin.getRange(nextBullRow,eventCol).setValue(eventEvent);
bulletin.getRange(nextBullRow,oppCol).setValue(eventOpp);
bulletin.getRange(nextBullRow,locCol).setValue(eventLocation);
nextBullRow++;
Logger.log(bulletin.getRange(nextBullRow-1,6).getValue());
if(bulletin.getRange(nextBullRow-1,6).getValue()=="Hollister") {
bulletin.getRange(nextBullRow-1,1,nextBullRow-1,6).setFontWeight("bold");
Logger.log("boop!");
}
'''
And here's the new version, which doesn't have the digital appetite for all the data;
'''
var compareDate = selDate;
var bulletin = ss.getSheetByName(reportSheetName);
var responses = ss.getSheetByName("Calendar");
var lastRow = responses.getLastRow(); //get last row of data sheet
var lastBull = bulletin.getLastRow(); //get last row of bulletin sheet
var nextBullRow = lastBull+1;
var nextBullItem = bulletin.getRange(nextBullRow,1);
var allData = responses.getRange(2, 1, responses.getLastRow(), 11).getValues();
for(var i = 2; i <= allData.getLastRow(); i++) {
var row = allData.getRange(i, 1, 1, 11).getValues();
Logger.log(row);
'''
Let me know if you want the whole function, I'll edit.
The question is: how can I speed data processing from the sluggish pace it is at currently. (Thanks Cooper!)
In your for loop you are calling getValues each iteration of your loop which looks like the issue. You've already got the data in your call above.
Try:
for(var i = 0; i <= allData.getLastRow(); i++) {
var row = allData[i];
Logger.log(row);
Had the same issue today. My solution was to duplicate the sheet and then work on the copy:
sheet=SpreadsheetApp.getActiveSpreadsheet()
copy=sheet.duplicateActiveSheet()
after duplicating, the copy is the active sheet, so you can directly work on it, e.g. delete rows like that:
sheet.deleteRows(1,2)
Related
The code below is overwriting on the existing data.
#OMila helped me with the original code, I could not articulate exactly what I needed hence starting a new question.
function Dom() {
var origin_sheet = SpreadsheetApp.getActive().getSheetByName('Dom_Sum');
var firstRow = 1;
var firstCol = 1;
var numRows = origin_sheet.getLastRow();
var numCols = 22;
var origin_values = origin_sheet.getRange(firstRow, firstCol, numRows, numCols).getValues();
var dest_values = [];
for(var i = 0; i < origin_values.length; i++) {
if(origin_values[i][0] != '') {
dest_values.push(origin_values[i]);
}
}
var dest_id = "1ZGq7L7bvF1INuDgZxhHnVsihkYkYubmncSAE5uC-Pq4";
var dest_sheet = SpreadsheetApp.openById(dest_id).getSheetByName("Master_Db");
var numRowsDest = dest_values.length;
var dest_range = dest_sheet.getRange(1, 1, numRowsDest, 22);
dest_range.setValues(dest_values);
}
I would like to add the data created in the "Dom_Sum" worksheet below the last row of data in the other workbook with the worksheet name "Master_Db"
#OMila I'm really grateful to you, and if you like we could offer you a consultation fee for future projects. (boseav#gmail.com)
Instead of writing your value into the range dest_sheet.getRange(1, 1, numRowsDest, numCols)
retrieve the last row of your destination sheet and write starting with the next row
var destLastRow=dest_sheet.getLastRow();
var dest_range = dest_sheet.getRange(destLastRow+1, 1, numRowsDest, numCols);
dest_range.setValues(dest_values);
The code below is overwriting on the existing data.
#OMila helped me with the original code, I could not articulate exactly what I needed hence starting a new question.
function Dom() {
var origin_sheet = SpreadsheetApp.getActive().getSheetByName('Dom_Sum');
var firstRow = 1;
var firstCol = 1;
var numRows = origin_sheet.getLastRow();
var numCols = 22;
var origin_values = origin_sheet.getRange(firstRow, firstCol, numRows, numCols).getValues();
var dest_values = [];
for(var i = 0; i < origin_values.length; i++) {
if(origin_values[i][0] != '') {
dest_values.push(origin_values[i]);
}
}
var dest_id = "1ZGq7L7bvF1INuDgZxhHnVsihkYkYubmncSAE5uC-Pq4";
var dest_sheet = SpreadsheetApp.openById(dest_id).getSheetByName("Master_Db");
var numRowsDest = dest_values.length;
var dest_range = dest_sheet.getRange(1, 1, numRowsDest, 22);
dest_range.setValues(dest_values);
}
I would like to add the data created in the "Dom_Sum" worksheet below the last row of data in the other workbook with the worksheet name "Master_Db"
#OMila I'm really grateful to you, and if you like we could offer you a consultation fee for future projects. (boseav#gmail.com)
Instead of writing your value into the range dest_sheet.getRange(1, 1, numRowsDest, numCols)
retrieve the last row of your destination sheet and write starting with the next row
var destLastRow=dest_sheet.getLastRow();
var dest_range = dest_sheet.getRange(destLastRow+1, 1, numRowsDest, numCols);
dest_range.setValues(dest_values);
I'm trying to run an IF function to match the date in the first column to "last month" and the date in the last column to "newest date" and copy and paste all of the rows matching this criteria (excluding the first and last column) to the bottom of the list.
This is the script I'm running and it isn't finding any matches when I know for a fact there are at least 100 rows matching this criteria:
function myFunction() {
var MCS = SpreadsheetApp.openById('[ID REMOVED FOR THIS Q]');
var MRB = MCS.getSheetByName('Media Rates Back');
var MRBrange = MRB.getRange(1,1,MRB.getLastRow(),1).getValues();
var dest = MRBrange.filter(String).length + 1;
var LM = new Date();
LM.setDate(1);
LM.setMonth(LM.getMonth()-1);
var LMs = Date.parse(LM);
var Datenew = MRB.getRange(MRB.getLastRow(),MRB.getLastColumn()).getValue();
var Datecol = MRB.getRange(1,6,MRB.getLastRow(),1).getValues();
var Datenews = Date.parse(Datenew);
for(var i=0; i<MRBrange.length; i++) {
if(Date.parse(MRBrange[i])==LMs && Date.parse(Datecol[i])==Datenews ) {
var NewRange = MRB.getRange(i,2,(MRB.getLastRow()-i),5);
var NewRangeV = NewRange.getValues();
var destination = MRB.getRange(MRB.getLastRow()+1,2);
Logger.log(NewRange);
NewRange.copyTo(destination);
}else{
Logger.log(i);
}
}}
Any help would be appreciated!
Rather than get the columns as separate ranges, I would get the entire range as one array, then loop over that and check the two columns.
I'm also assuming your values are formatted as dates in the Sheet, in which case you don't need to use Date.parse(), and that your actual date logic is correct.
You can try using the debugger and set a breakpoint at the IF, so you can check the values it is comparing. or put a Logger.log call to list your comparisons.
var last_month_column = 1;
var newest_date_column = MRB.getLastColumn();
var MRBrange = MRB.getRange(1,1,MRB.getLastRow(),newest_date_column).getValues();
for(var row in MRBrange) {
if(MRBrange[row][last_month_column]==LMs && Datecol[row][newest_date_column] ==Datenews ) {
/* your copy logic here */
}else{
Logger.log(i);
}
}
I think the problem may be that MRBrange is a 2d Array. So I used another loop to convert it to a 1d array.
function myFunction() {
var MCS = SpreadsheetApp.openById('[ID REMOVED FOR THIS Q]');
var MRB = MCS.getSheetByName('Media Rates Back');
var MRBrangeA = MRB.getRange(1,1,MRB.getLastRow(),1).getValues();//2d array
var MRBrange=[];
for(var i=0;i<MRBrangeA.length;i++)
{
MRBrange.push(MRBrangA[i][0]);//1d array
}
var dest = MRBrange.filter(String).length + 1;
var LM = new Date();//current day
LM.setDate(1);//first day of month
LM.setMonth(LM.getMonth()-1);//first day of last month
var LMs = Date.parse(LM);
var Datenew = MRB.getRange(MRB.getLastRow(),MRB.getLastColumn()).getValue();
var Datecol = MRB.getRange(1,6,MRB.getLastRow(),1).getValues();
var Datenews = Date.parse(Datenew);
for(var i=0; i<MRBrange.length; i++) {
if(Date.parse(MRBrange[i])==LMs && Date.parse(Datecol[i])==Datenews ) {
var NewRange = MRB.getRange(i,2,(MRB.getLastRow()-i),5);
var NewRangeV = NewRange.getValues();
var destination = MRB.getRange(MRB.getLastRow()+1,2);
Logger.log(NewRange);
NewRange.copyTo(destination);
}else{
Logger.log(i);
}
}}
I currently have a list with two columns. The first column is student name, and the second column is the number of points they have.
I imported this list from multiple spreadsheets so there were many duplicates on the names of the students. I am able to remove the duplicates, but I want to keep a tally on the total points they have. For example:
Amy 10
Bob 9
Carol 15
Amy 12
would turn into:
Amy 22
Bob 9
Carol 15
This is what I have so far:
var target = SpreadsheetApp.getActiveSpreadsheet();
var sheet = target.getSheetByName("Sheet2");
var data = sheet.getRange("A2:B1000").getValues();
var newData = new Array();
var k = 0
var finallist = []
for(i in data){
k++;
var row = data[i];
var duplicate = false;
for(j in newData){
if(row[0] == newData[j][0]){
duplicate = true;
var storedHour = sheet.getRange("B"+k).getValue();
var position = finallist.indexOf(row[0]);
var originalCell = sheet.getRange("B"+(position+1));
var originalHour = originalCell.getValue();
originalCell.setValue(originalHour + storedHour);
sheet.getRange(k,2).setValue("")
sheet.getRange(k,1).setValue("")
}
}
if(!duplicate){
newData.push(row);
finallist.push(row[0])
}
}
}
The problem I'm having is that we have a really large data sample and I'm afraid it may run over Google's 5 minute maximum execution time. Is there another more efficient way to achieve my goal?
Your code is running slow because Spreadsheets API methods (like getRange) are time consuming and much slower then other JavaScript code.
Here is optimized function with reduced number of such Spreadsheets API calls:
function calcNumbers()
{
var target = SpreadsheetApp.getActiveSpreadsheet();
var sheet = target.getSheetByName("Sheet2");
var lastRow = sheet.getLastRow();
var dataRange = sheet.getRange(2, 1, lastRow-1, 2);
var data = dataRange.getValues();
var pointsByName = {};
for (var i = 0; i < data.length; i++)
{
var row = data[i];
var curName = row[0];
var curNumber = row[1];
// empty name
if (!curName.trim())
{
continue;
}
// if name found first time, save it to object
if (!pointsByName[curName])
{
pointsByName[curName] = Number(curNumber);
}
// if duplicate, sum numbers
else
{
pointsByName[curName] += curNumber;
}
}
// prepare data for output
var outputData = Object.keys(pointsByName).map(function(name){
return [name, pointsByName[name]];
});
// clear old data
dataRange.clearContent();
// write calculated data
var newDataRange = sheet.getRange(2, 1, outputData.length, 2);
newDataRange.setValues(outputData);
}
Sorting before comparing allows looking at the next item only instead of all items for each iteration. A spillover benefit is finallist result is alphabatized. Execution time reduction significant.
function sumDups() {
var target = SpreadsheetApp.getActiveSpreadsheet();
var sheet = target.getSheetByName("Sheet2");
var data = sheet.getRange("A2:B" + sheet.getLastRow()).getValues().sort();
var finallist = [];
for(var i = 0; i<= data.length - 1; i++){
var hours = data[i][1];
while((i < data.length - 1) && (data[i][0] == data[i+1][0])) {
hours += data[i+1][1];
i++;
};
finallist.push([data[i][0], hours]);
};
Logger.log(finallist);
}
Edit: the simple data structure with the name being in the first column allows this to work. For anything more complex understanding and applying the methods shown in #Kos's answer is preferable
When I run my (messy) script it seems to run one extra time than required. i.e. below the last row it creates a pdf and marks a cell as processed.
The second issue is that the URLs don't seem to line up correctly with the name that it should be.
I appreciate the code is very messy but I'm happy to explain the reasoning for any part if it doesn't make sense.
Thanks in advance for any help!
The spreadsheet can be found here:
https://docs.google.com/spreadsheets/d/1nq5RRcwAKk9sFm6jVypFT_qJWcOGxFDZRv6ZpB-QClw/edit#gid=1194576382
and the code that's not working as expected is:
var ss = SpreadsheetApp.getActiveSpreadsheet()
var rawData = "rawData"
var practicePivot = "practicePivot"
var querySheet = "querySheet"
var pdfSheet = "pdfSheet"
var contactList = "contactList"
function createPDF(){
var sourceSheet = ss.getSheetByName("querySheet")
var pdfList = ss.getSheetByName("practicePivot")
var contactList = ss.getSheetByName("contactList")
var sourceRow = sourceSheet.getLastRow()
var sourceColumn = sourceSheet.getLastColumn()
var sourceStartRow = 4 //skips the headers and only pulls query data
var sourceStartColumn = 1
var sourceRange = sourceSheet.getRange(sourceStartRow, sourceStartColumn, sourceRow, sourceColumn)
var sourceValues = sourceRange.getValues()
var pdfLastRow = pdfList.getLastRow()
var storePracticeName = sourceSheet.getRange("A2").getValues()
var newSpreadsheet = SpreadsheetApp.create("Summary of Patients for" +storePracticeName)
sourceSheet.copyTo(newSpreadsheet, {contentsOnly: true})
newSpreadsheet.getSheetByName("sheet1").activate()
newSpreadsheet.deleteActiveSheet()
var pdfURLtemp = DriveApp.getFileById(newSpreadsheet.getId()).getAs("application/pdf")
var pdf = DriveApp.createFile(pdfURLtemp)
var pdfURL = pdf.getUrl()
Logger.log(pdfURL)
return pdfURL
}
function createQuery()
{
//Duplication check
var pdfCreated = "pdfCreated";
var pdfEmailed = "pdfEmailed";
var sheet = ss.getSheetByName("practicePivot");
var startRow = 2;
var lastRow = sheet.getLastRow();
var lastColumn = sheet.getLastColumn();
var dataRange = sheet.getRange(startRow, 1, lastRow, lastColumn) ;
var data = dataRange.getValues();
for (var i = 0; i < data.length; ++i)
{
var row = data[i];
var practiceName = row[0]
var pdfCheck = row[2]
var copySelection = sheet.getRange(startRow + i, 1)
var copyData = copySelection.getValues()
var copyLocation = ss.getSheetByName("querySheet")
var copyCell = copyLocation.getRange("A2")
if (pdfCheck != pdfCreated)
{
var pdfURL = createPDF()
copyCell.copyTo(copyData)
sheet.getRange(startRow + i, 4).setValue(pdfURL)
sheet.getRange(startRow + i, 3).setValue(pdfCreated)
SpreadsheetApp.flush()
}
}
}
Your start row is 2:
var startRow = 2;
Your loop is looping until the count of the last row. So if there were 10 rows, and the data starts in row 2, then you need the loop to run 9 times. But your loop is running 10 times.
So you need to adjust the stop for the loop.
var L = data.length - 1;//The number of rows in the data - not the sheet
for (var i = 0; i < L; ++i)