Error: Data between close double quote (") and field separator - javascript

I'm trying to use Google Apps Script to take a CSV from Google Drive and put it into Big Query. When I upload, I get this error:
"Error while reading data, error message: Error detected while parsing row starting at position: 560550. Error: Data between close double quote (") and field separator."
I've tried looking at that byte position of the file and its way outside the bounds of the CSV (it only goes to ~501500 bytes).
Here's a link to the CSV that I'm using which is a scrape of a website: https://drive.google.com/file/d/1k3cGlTSA_zPQCtUkt20vn6XKiLPJ7mFB/view?usp=sharing
Here's my relevant code:
function csvToBigQuery(exportFolder, csvName, bqDatasetId){
try{
//get most recent export from Screaming Frog
var mostRecentFolder = [];
while(exportFolder.hasNext()){
var folder = exportFolder.next();
var lastUpdated = folder.getLastUpdated();
if(mostRecentFolder.length == 0)
mostRecentFolder = [folder.getLastUpdated(),folder.getId()];
else if(lastUpdated > mostRecentFolder[0])
mostRecentFolder = [lastUpdated, folder.getId()];
}
var folderId = mostRecentFolder[1];
var file = DriveApp.getFolderById(folderId).getFilesByName(csvName + '.csv').next();
if(!file)
throw "File doesn't exist";
//get csv and add date column.
//getBlob().getDataAsString().replace(/(["'])(?:(?=(\\?))\2[\s\S])*?\1/g, function(e){return e.replace(/\r?\n|\r/g, ' ')})
var rows = Utilities.parseCsv(file.getBlob().getDataAsString());
Logger.log(rows);
var numColumns = rows[0].length;
rows.forEach(function(row){
row[numColumns] = date;
});
rows[0][numColumns] = 'Date';
let csvRows = rows.map(values =>values.map(value => JSON.stringify(value).replace(/\\"/g, '""')));
let csvData = csvRows.map(values => values.join(',')).join('\n');
//log(csvData)
var blob = Utilities.newBlob(csvData, 'application/octet-stream');
//create job for inserting to BQ.
var loadJob = {
configuration: {
load: {
destinationTable: {
projectId: bqProjectId,
datasetId: bqDatasetId,
tableId: csvName
},
autodetect: true, // Infer schema from contents.
writeDisposition: 'WRITE_APPEND',
}
}
};
//append to table in BQ.
BigQuery.Jobs.insert(loadJob, bqProjectId, blob);
}catch(e){
Logger.log(e);
}
}

Modification points:
From your error message, I thought that there might be the parts which are not enclosed by the double quota. So, I searched When I saw your CSV data and your CSV data is replaced \"(|.+?)\" with "" using the following script, it was found that the row 711 has the value.
function sample() {
var id = "###"; // File ID of your CSV file.
// This is your script.
var file = DriveApp.getFileById(id);
var rows = Utilities.parseCsv(file.getBlob().getDataAsString());
var numColumns = rows[0].length;
var date = "sample";
rows.forEach(function(row){
row[numColumns] = date;
});
rows[0][numColumns] = 'Date';
let csvRows = rows.map(values =>values.map(value => JSON.stringify(value).replace(/\\"/g, '""')));
let csvData = csvRows.map(values => values.join(',')).join('\n');
// I added below script for checking your CSV data.
var res = csvData.replace(/\"(|.+?)\"/g, "");
DriveApp.createFile("sample.txt", res);
}
The row 711 is as follows.
"https://supergoop.com/products/lip-shield-trio/?utm_source=Gorgias&utm_medium=CustomerCare&utm_campaign=crosssellhello\","text/html; charset=utf-8","200","OK","Non-Indexable","Canonicalised","Lip Shield Trio - Restores, Protects + Water-resistant – Supergoop!","67","595","Moisturizing lip protection made from antioxidant-rich coconut, avocado, and grape seed oil.","92","576","","0","Lip Shield Trio","15","Lip Shield Trio","15","Why We Love It","14","Ingredients","11","","","","https://supergoop.com/products/lip-shield-trio","","","","","451488","754","1.686","5","","12","4","0.590","205","80","8","5","","","","","f6d1476960d22b1c5964581e161bdd49","0.064","","","","","HTTP/1.1","https://supergoop.com/products/lip-shield-trio/?utm_source=Gorgias&utm_medium=CustomerCare&utm_campaign=crosssellhello%5C"
From this value, I found that \" is used at "https://supergoop.com/products/lip-shield-trio/?utm_source=Gorgias&utm_medium=CustomerCare&utm_campaign=crosssellhello\". I thought that the reason of your issue might be due to this.
So in order to avoid this issue, how about the following modification?
Modified script:
From:
let csvRows = rows.map(values =>values.map(value => JSON.stringify(value).replace(/\\"/g, '""')));
To:
let csvRows = rows.map(values =>values.map(value => JSON.stringify(value).replace(/\\"/g, '""').replace(/\\"/g, '')));
or
From:
var rows = Utilities.parseCsv(file.getBlob().getDataAsString());
To:
var rows = Utilities.parseCsv(file.getBlob().getDataAsString().replace(/\\/g, ''));
By this modification, I could confirm that the file size was reduced with 2 bytes between your script and the modified script. And also, when above check script is used for the CSV data using the modified script, I could confirm that all rows have no values.

Related

Invalid argument: replacement error when populating google doc using google script

I have written a code to populate data from a spreadsheet into a google doc and save it to drive using g-sript. Here is the code for the same :
function onOpen() {
const ui = SpreadsheetApp.getUi();
const menu = ui.createMenu('Invoice creator');
menu.addItem('Generate Invoice', 'invoiceGeneratorFunction');
menu.addToUi();
}
function invoiceGeneratorFunction() {
const invoiceTemplate = DriveApp.getFileById('125NPu-n77F6N8hez9w63oSzbWrtryYpRGOkKL3IbxZ8');
const destinationFolder = DriveApp.getFolderById('163_wLsNGkX4XDUiSOcQ88YOPe3vEx7ML');
const sheet_invoice = SpreadsheetApp.getActiveSpreadsheet().getSheetByName('New Invoice Sheet');
const rows = sheet_invoice.getDataRange().getValues();
Logger.log(rows);
rows.forEach(function(row, index) {
if (index === 0) return;
if (row[12] != "") return;
const copy = invoiceTemplate.makeCopy(`${row[1]} VIN Number: ${row[2]}`,destinationFolder);
const doc = DocumentApp.openById(copy.getId());
const body = doc.getBody();
var friendlyDateBilled = new Date(row[0]).toLocaleDateString();
var friendlyDateDelivery = new Date(row[3]).toLocaleDateString();
body.replaceText('{{Date Billed}}',friendlyDateBilled);
body.replaceText('{{Customer Name}}',row[1]);
body.replaceText('{{VIN Number}}',row[2]);
body.replaceText('{{Date of Delivery}}',friendlyDateDelivery);
body.replaceText('{{Package}}',rows[4]);
body.replaceText('{{Price}}',rows[5]);
body.replaceText('{{Output CGST}}',rows[6]);
body.replaceText('{{Output SGST}}',rows[7]);
body.replaceText('{{Discount}}',rows[8]);
body.replaceText('{{Total Price}}',rows[9]);
body.replaceText('{{Balance}}',rows[10]);
body.replaceText('{{Remarks}}',rows[11]);
doc.saveAndClose();
const url = doc.getUrl();
sheet_invoice.getRange(index+1, 13).setValue(url);
})
}
I have created a menu button for the script to run. But when i run it I get an error saying :
Exception: Invalid argument: replacement
at unknown function
at invoiceGeneratorFunction(Code:17:8)
(Here line 32 is body.replaceText('{{Package}}',rows[4]);
and line 17 is the start of forEach)
Interestingly when I comment out the rest of body.replaceText lines after that line, the code works. I can't understand what the problem is, if it's working if I comment out the lines.
In your script, rows is 2 dimensional array retrieved with sheet_invoice.getDataRange().getValues(). When I saw your loop, after the line of body.replaceText('{{Package}}',rows[4]);, rows is used. In this case, rows[4] is 1-dimensional array. It is required to be the string for the arguments of replaceText(searchPattern, replacement). I think that this might be the reason for your issue. In order to remove this issue, how about the following modification?
From:
body.replaceText('{{Package}}',rows[4]);
body.replaceText('{{Price}}',rows[5]);
body.replaceText('{{Output CGST}}',rows[6]);
body.replaceText('{{Output SGST}}',rows[7]);
body.replaceText('{{Discount}}',rows[8]);
body.replaceText('{{Total Price}}',rows[9]);
body.replaceText('{{Balance}}',rows[10]);
body.replaceText('{{Remarks}}',rows[11]);
To:
body.replaceText('{{Package}}',row[4]);
body.replaceText('{{Price}}',row[5]);
body.replaceText('{{Output CGST}}',row[6]);
body.replaceText('{{Output SGST}}',row[7]);
body.replaceText('{{Discount}}',row[8]);
body.replaceText('{{Total Price}}',row[9]);
body.replaceText('{{Balance}}',row[10]);
body.replaceText('{{Remarks}}',row[11]);
Note:
I'm not sure about your actual values of rows. So I'm not sure whether the values of row[4] to row[11] are what you want. If those values are not the values you expect, please check your Spreadsheet again.
Reference:
replaceText(searchPattern, replacement)

Not able to write splitted text in json file in cypress

I am testing an application where the alert text says like
Customer added successfully with customer id :13
Now i want to grab 13 and save it in json file.
Somehow its not getting written in json file.
Please help
Below is my code
export function stripcustomerid() {
cy.on('window:alert', (txt) => {
cy.readFile('cypress/fixtures/account_details.json').then((data) => {
var customerno = txt.split(':')
const cno = customerno[1]
data.customerid = cno
cy.writeFile(
'cypress/fixtures/account_details.json',
JSON.stringify(data)
)
})
})
}
Assuming that the txt variable has the string Customer added successfully with customer id :13 and also your account_details.json file has an existing customer id data like customerid: 55, because you are changing this value. Then your code looks correct.
You can replace these three lines:
var customerno = txt.split(':')
const cno = customerno[1]
data.customerid = cno
with
data.customerid = txt.split(':')[1]

How do you loop through an object and replace text

I have a script that should create a pdf file from a google form submission and grabs the data to be changed as an object. However I am using the replaceText action to make the changes to the doc and I'm getting the following error.
Exception: Invalid argument: replacement
at Create_PDF(Code:37:8)
at After_Submit(Code:13:19)
It is supposed to change the values in the generated doc file and it worked when I used the namedValues function. However now that I'm using range instead it doesn't seem to work.
function After_Submit(e){
var range = e.range;
var row = range.getRow(); //get the row of newly added form data
var sheet = range.getSheet(); //get the Sheet
var headers = sheet.getRange(1, 1, 1, 129).getValues().flat(); //get the header names from A-O
var data = sheet.getRange(row, 1, 1, headers.length).getValues(); //get the values of newly added form data + formulated values
var values = {}; // create an object
for( var i = 0; i < headers.length; i++ ){
values[headers[i]] = data[0][i]; //add elements to values object and use headers as key
}
Logger.log(values);
const pdfFile = Create_PDF(values);
sendEmail(e.namedValues['Email Address to Receive File '][0],pdfFile);
}
function sendEmail(email,pdfFile){
GmailApp.sendEmail(email, "Subject", "Files Attached", {
attachments: [pdfFile],
name: "From Email"
});
}
function Create_PDF(values) {
const PDF_folder = DriveApp.getFolderById("ID_1");
const TEMP_FOLDER = DriveApp.getFolderById("ID_2");
const PDF_Template = DriveApp.getFileById('ID_3');
const newTempFile = PDF_Template.makeCopy(TEMP_FOLDER);
const OpenDoc = DocumentApp.openById(newTempFile.getId());
const body = OpenDoc.getBody();
console.log(body);
body.replaceText("{{Timestamp}}", values['Timestamp'][0]);
body.replaceText("{{Location}}", values['Location'][0]);
body.replaceText("{{Item1}}", values['Item1'][0]);
body.replaceText("{{Item2}}", values['Item2'][0]);
body.replaceText("{{Itme3}}", values['Item3'][0]);
body.replaceText("{{e1}}", values['e1'][0]);
body.replaceText("{{e2}}", values['e2'][0]);
body.replaceText("{{e3}}", values['e3'][0]);
body.replaceText("{{e4}}", values['e4'][0]);
body.replaceText("{{e5}}", values['e5'][0]);
body.replaceText("{{e6}}", values['e6'][0]);
body.replaceText("{{e7}}", values['e7'][0]);
body.replaceText("{{e8}}", values['e8'][0]);
body.replaceText("{{e9}}", values['e9'][0]);
body.replaceText("{{e10}}", values['e10'][0]);
body.replaceText("{{e11}}", values['e11'][0]);
body.replaceText("{{e12}}", values['e12'][0]);
body.replaceText("{{e13}}", values['e13'][0]);
body.replaceText("{{e14}}", values['e14'][0]);
body.replaceText("{{e15}}", values['e15'][0]);
body.replaceText("{{e16}}", values['e16'][0]);
body.replaceText("{{e17}}", values['e17'][0]);
body.replaceText("{{e18}}", values['e18'][0]);
body.replaceText("{{e19}}", values['e19'][0]);
body.replaceText("{{e20}}", values['e20'][0]);
body.replaceText("{{e21}}", values['e21'][0]);
body.replaceText("{{e22}}", values['e22'][0]);
body.replaceText("{{e23}}", values['e23'][0]);
body.replaceText("{{e24}}", values['e24'][0]);
body.replaceText("{{e25}}", values['e25'][0]);
body.replaceText("{{e26}}", values['e26'][0]);
body.replaceText("{{e27}}", values['e27'][0]);
body.replaceText("{{e28}}", values['e28'][0]);
body.replaceText("{{e29}}", values['e29'][0]);
body.replaceText("{{e30}}", values['e30'][0]);
body.replaceText("{{e31}}", values['e31'][0]);
body.replaceText("{{e32}}", values['e32'][0]);
body.replaceText("{{e33}}", values['e33'][0]);
body.replaceText("{{e34}}", values['e34'][0]);
body.replaceText("{{e35}}", values['e35'][0]);
body.replaceText("{{e36}}", values['e36'][0]);
body.replaceText("{{e37}}", values['e37'][0]);
body.replaceText("{{e38}}", values['e38'][0]);
body.replaceText("{{e39}}", values['e39'][0]);
body.replaceText("{{H1}}", values['H1'][0]);
body.replaceText("{{H2}}", values['H2'][0]);
body.replaceText("{{H3}}", values['H3'][0]);
body.replaceText("{{H4}}", values['H4'][0]);
body.replaceText("{{H5}}", values['H5'][0]);
body.replaceText("{{H6}}", values['H6'][0]);
body.replaceText("{{H7}}", values['H7'][0]);
body.replaceText("{{H8}}", values['H8'][0]);
body.replaceText("{{H9}}", values['H9'][0]);
body.replaceText("{{H10}}", values['H10'][0]);
body.replaceText("{{H11}}", values['H11'][0]);
body.replaceText("{{H12}}", values['H12'][0]);
body.replaceText("{{H13}}", values['H13'][0]);
body.replaceText("{{H14}}", values['H14'][0]);
OpenDoc.saveAndClose();
const BLOBPDF = newTempFile.getAs(MimeType.PDF);
const pdfFile = PDF_folder.createFile(BLOBPDF).setName("FLHA");
console.log("The file has been created ");
return pdfFile;
}
Your question was how to loop through an object and replace text
This creates an object from Sheet0:
Sheet0:
one
pattern
two
this is the pattern
three
pattern pattern
four
nothing
five
nothing
Code:
function replacepattern() {
const ss = SpreadsheetApp.getActive();
const sh = ss.getSheetByName('Sheet0');
const vs = sh.getRange(1,1,sh.getLastRow(), 2).getValues();
//creating object from spreadsheet
let obj = {pA:[]};
vs.forEach(r =>{
obj[r[0]]=r[1];
obj.pA.push(r[0]);
});
Logger.log(JSON.stringify(obj));
let oA = obj.pA.map(p => [obj[p].replace(/pattern/g,'replacement')]);//doing the replacement in an object
sh.getRange(1,sh.getLastColumn() + 1,oA.length, oA[0].length).setValues(oA);//outputting the replaced string in the next column
Logger.log(JSON.stringify(oA));
}
Sheet0 after running once:
one
pattern
replacement
two
this is the pattern
this is the replacement
three
pattern pattern
replacement replacement
four
nothing
nothing
five
nothing
nothing
This is related to my answer here.
The error code Exception: Invalid argument: replacement at Create_PDF(Code:37:8) at After_Submit(Code:13:19) is caused by the null value of values['Timestamp'][0]. If you try to print the data type of values['Timestamp'], it will return a type object, since that object does not have value for index 0 to it will return a null value.
For entries that are type String, if you add [0] to it, it will return only the first element of the string. Example you have "Test" string, adding [0] to it will return "T"
To fix that, just remove the [0] in all of body.replaceText(..., values['...'][0]) entries.
OR
Loop through values object by replacing the body.replaceText entries in your code with this:
for (const key in values) {
body.replaceText("{{"+key+"}}", values[key]);
}
Example usage:
Form inputs:
Output:
Reference:
JavaScript for..in

Appending a new row with only formulas using Sheets API v4

I'm pretty new to Sheets API and get a lot of bugs.
I would like to append a new row to sheet based on last row. This would include copying the format and pasting formulas with an autofill but not the values.
Here what I've came up using app script.
I'm sure I'm not using the best way so for the moment I've
retrieved formulas from range SUCCESS
tried using autoFillRequest to populate next row with 10 columns(just a try). FAILED
I've put in comment the getFormulas-like request and show you what I have for the moment with the autoFill request.
I get the following error:
Invalid JSON payload received. Unknown name "source_and_destination" at 'requests[0]': Cannot find field.
function insertNewRow(){
var ssId = "my_spreadsheet_id"
/*var params = {
ranges: ['Feuille1!21:21'],
valueRenderOption: 'FORMULA'
};
var values = Sheets.Spreadsheets.Values.batchGet(ssId, params);
var valueRange = Sheets.newValueRange();
valueRange.majorDimension = "ROWS";
valueRange.values = values.valueRanges[0].values;
Logger.log(values.valueRanges[0].values[0].length);
valueRange.range= 'Feuille1!22:22'
//var req = Sheets.Spreadsheets.Values.update(valueRange , ssId, 'Feuille1!22:22', {
// valueInputOption: 'USER_ENTERED'
//})*/
var AFreq = Sheets.newAutoFillRequest();
AFreq.range = Sheets.newGridRange();
AFreq.range.startRowIndex = 1;
AFreq.range.startColumnIndex = 0;
AFreq.range.endRowIndex = 2;
AFreq.range.endColumnIndex = 10;
AFreq.range.sheetId = 0;
AFreq.sourceAndDestination = Sheets.newSourceAndDestination();
AFreq.sourceAndDestination.dimension = "ROWS";
AFreq.sourceAndDestination.fillLength = 10;
AFreq.sourceAndDestination.source = Sheets.newGridRange();
AFreq.sourceAndDestination.source.startRowIndex = 0;
AFreq.sourceAndDestination.source.startColumnIndex = 0;
AFreq.sourceAndDestination.source.endColumnIndex = 10
AFreq.sourceAndDestination.source.endRowIndex = 1;
AFreq.sourceAndDestination.source.sheetId = 0;
var req = Sheets.newBatchUpdateSpreadsheetRequest();
req.requests = [AFreq];
Sheets.Spreadsheets.batchUpdate(req, ssId);
}
Tell me if I'm wrong but I though about separating the tasks into multiple requests
grab the formulas
insert new row
copy/paste preceding fromat to new row
pasting formulas
Am I going in the right direction?
Any help is greatly appreciated.
Issues:
Request object is missing in Request body.
AutoFillRequest has two union area fields, whereas exactly one is acceptable.
Empty range selection in GridRange.
Solution:
Fix syntax errors mentioned above
Used plain JSON request body to easily identify such errors
Sample Script:
function autoFill() {
var ssId = 'my_spreadsheet_id';
var req = {//request body
requests: [ //requests array
{//request Object
autoFill: { //autoFill request
//range OR sourceAndDestination;
//equal to selecting Sheet1!A1:J10 and clicking autoFill from menu
range: {//GridRange
sheetId: 0,
startRowIndex: 0,
endRowIndex: 10, //end exclusive
startColumnIndex: 0,
endColumnIndex: 10,
},
},
},
],
};
Sheets.Spreadsheets.batchUpdate(req, ssId);
}
References:
RequestBody
AutoFillRequest
GridRange

Apps Script, convert a Sheet range to Blob

Background:
I'm trying to upload an individual row of data from a Google Sheet and append it to a BigQuery table.
Method: I've been using https://developers.google.com/apps-script/advanced/bigquery to do this, but instead of a file of data as the example is, I am using my own sheet with data from a specific row:
var file = SpreadsheetApp.getActiveSpreadsheet();
var currentSheet = file.getSheetByName(name);
var lastRow = currentSheet.getLastRow()
var lastC = currentSheet.getLastColumn()
var rows = currentSheet.getRange(2,1,1,lastC).getValues();
"rows" is the row of data to be imported to BQ. I've tried a multitude of things, and according to another StackOverflow question, "rowsCSV" makes the 2D array of values CSV.
var rowsCSV = rows.join("\n");
var data = rowsCSV.getBlob().setContentType('application/octet-stream');
Problem: Every time I run the function, I get the error "Cannot find function getBlob in object Blob. " or, "Cannot convert Array to (class)[][]", or "Cannot find function getBlob in object Tue May 16 2017 00:00:00 GMT+0200 (CEST),58072.4,,,,,,,,,,,test ", where the last bit ("Tue May..") is the actual data of the row.
What am I doing wrong here?
There is no getBlob method for an array. You will have to use the Utilities.newBlob() to get your blob from a string. You can find the documentation on the same here
var rowsCSV = rows.join("\n");
var blob = Utilities.newBlob(rowsCSV, "text/csv")
Logger.log(blob.getDataAsString())
var data = blob.setContentType('application/octet-stream');
Equivalently you can do this
var rowsCSV = rows.join("\n");
var data = Utilities.newBlob(rowsCSV, 'application/octet-stream')
For anyone else viewing this, Jack Brown's answer is correct, you just need to change
var rows = currentSheet.getRange(2,1,1,lastC).getValues();
to
var rows = currentSheet.getRange(2,1,lastRow,lastC).getValues();
Based on the correction given by #JackBrown I have edited my code, however unable to push data to the Big Query. The below code create table, but don't push values.
/**
* Loads a CSV into BigQuery
*/
function loadCsv() {
// Replace this value with the project ID listed in the Google
// Cloud Platform project.
var projectId = 'master-ad-data';
// Create a dataset in the BigQuery UI (https://bigquery.cloud.google.com)
// and enter its ID below.
var datasetId = 'DataImportFromSpreadsheet';
// Sample CSV file of Google Trends data conforming to the schema below.
// https://docs.google.com/file/d/0BwzA1Orbvy5WMXFLaTR1Z1p2UDg/edit
var csvFileId = '17kYH6hP2RlsFeUmwM1v6WOgm2FKrwLTXWDhA2ZLISN8';
var name = 'Sheet1';
// Create the table.
var tableId = 'pets_' + new Date().getTime();
var table = {
tableReference: {
projectId: projectId,
datasetId: datasetId,
tableId: tableId
},
schema: {
fields: [
{name: 'CampaignLabels', type: 'STRING'},
{name: 'ImpressionShare', type: 'INTEGER'}
]
}
};
table = BigQuery.Tables.insert(table, projectId, datasetId);
Logger.log('Table created: %s', table.id);
var file = SpreadsheetApp.getActiveSpreadsheet();
var currentSheet = file.getSheetByName(name);
var lastRow = currentSheet.getLastRow()
var lastC = currentSheet.getLastColumn()
var rows = currentSheet.getRange(2,1,1,lastC).getValues();
var rowsCSV = rows.join("\n");
Logger.log("Check This"+" "+rowsCSV);
var data = Utilities.newBlob(rowsCSV, 'application/octet-stream')
// Create the data upload job.
var job = {
configuration: {
load: {
destinationTable: {
projectId: projectId,
datasetId: datasetId,
tableId: tableId
},
skipLeadingRows: 1
}
}
};
job = BigQuery.Jobs.insert(job, projectId, data);
Logger.log('Load job started. Check on the status of it here: ' +
'https://bigquery.cloud.google.com/jobs/%s', projectId);
}

Categories