We Keep Coding

Best to way to Run a Node JS script on Google Cloud

First of all, "I'm noob"
My first time running some script on a cloud server.
So, I have a code that counts the number of views from my youtube vídeo and writes it in the title.
But I want that code running 24/7.
I make it works running on a VM instance on google cloud using cron-node running the code every 5 minutes and make it run forever with "forever" on the node.
And my question is: Have a better way to run it? Like using cloud functions or something? Or I should still run it on VM instance?
If yes, what should I do? Because I don't know how to use external dependencies on Cloud function like google API, if I don't have a console to install googleapi dependencies. (Maybe it's easy to make it, but I just don't know yet).
var readline = require('readline');
var { google } = require('googleapis');
var OAuth2 = google.auth.OAuth2;
const cron = require("node-cron");
cron.schedule("*/5 * * * *", () => {
// If modifying these scopes, delete your previously saved credentials
// at ~/.credentials/youtube-nodejs-quickstart.json
var SCOPES = ['https://www.googleapis.com/auth/youtube.force-ssl'];
var TOKEN_DIR = (process.env.HOME || process.env.HOMEPATH ||
process.env.USERPROFILE) + '/.credentials/';
var TOKEN_PATH = TOKEN_DIR + 'update-youtube-title.json';
const youtube = google.youtube('v3');
const video_id = 'f0ARwVVxoBc';
// Load client secrets from a local file.
fs.readFile('credentials.json', function processClientSecrets(err, content) {
if (err) {
console.log('Error loading client secret file: ' + err);
return;
}
// Authorize a client with the loaded credentials, then call the YouTube API.
authorize(JSON.parse(content), makeAuthCall);
});
/**
* Create an OAuth2 client with the given credentials, and then execute the
* given callback function.
*
* #param {Object} credentials The authorization client credentials.
* #param {function} callback The callback to call with the authorized client.
*/
function authorize(credentials, callback) {
var clientSecret = credentials.installed.client_secret;
var clientId = credentials.installed.client_id;
var redirectUrl = credentials.installed.redirect_uris[0];
var oauth2Client = new OAuth2(clientId, clientSecret, redirectUrl);
// Check if we have previously stored a token.
fs.readFile(TOKEN_PATH, function (err, token) {
if (err) {
getNewToken(oauth2Client, callback);
} else {
oauth2Client.credentials = JSON.parse(token);
callback(oauth2Client);
}
});
}
/**
* Get and store new token after prompting for user authorization, and then
* execute the given callback with the authorized OAuth2 client.
*
* #param {google.auth.OAuth2} oauth2Client The OAuth2 client to get token for.
* #param {getEventsCallback} callback The callback to call with the authorized
* client.
*/
function getNewToken(oauth2Client, callback) {
var authUrl = oauth2Client.generateAuthUrl({
access_type: 'offline',
scope: SCOPES
});
console.log('Authorize this app by visiting this url: ', authUrl);
var rl = readline.createInterface({
input: process.stdin,
output: process.stdout
});
rl.question('Enter the code from that page here: ', function (code) {
rl.close();
oauth2Client.getToken(code, function (err, token) {
if (err) {
console.log('Error while trying to retrieve access token', err);
return;
}
oauth2Client.credentials = token;
storeToken(token);
callback(oauth2Client);
});
});
}
/**
* Store token to disk be used in later program executions.
*
* #param {Object} token The token to store to disk.
*/
function storeToken(token) {
try {
fs.mkdirSync(TOKEN_DIR);
} catch (err) {
if (err.code != 'EEXIST') {
throw err;
}
}
fs.writeFile(TOKEN_PATH, JSON.stringify(token), (err) => {
if (err) throw err;
console.log('Token stored to ' + TOKEN_PATH);
});
}
/**
* Lists the names and IDs of up to 10 files.
*
* #param {google.auth.OAuth2} auth An authorized OAuth2 client.
*/
/*
function getChannel(auth) {
var service = google.youtube('v3');
service.channels.list({
auth: auth,
part: 'snippet,contentDetails,statistics',
forUsername: 'GoogleDevelopers'
}, function (err, response) {
if (err) {
console.log('The API returned an error: ' + err);
return;
}
var channels = response.data.items;
if (channels.length == 0) {
console.log('No channel found.');
} else {
console.log('This channel\'s ID is %s. Its title is \'%s\', and ' +
'it has %s views.',
channels[0].id,
channels[0].snippet.title,
channels[0].statistics.viewCount);
}
});
}
*/
//código novo
const makeAuthCall = (auth) => {
//get views
youtube.videos.list(
{
auth: auth,
id: video_id,
part: 'snippet,statistics'
},
(err, response) => {
if (err) {
console.log(`A contagem de views falhou ${err}`)
return;
}
if (response.data.items[0]) {
//atualização encontrada
console.log('Encontramos uma atualização...');
updateVideoTitle(response.data.items[0], auth);
}
}
);
};
const updateVideoTitle = (video, auth) => {
let viewsText = video.statistics.viewCount.toLocaleString();
const newTitle = `Esse Vídeo tem ${viewsText} Visualizações`;
const oldTitle = video.snippet.title;
if (newTitle != oldTitle){
video.snippet.title = newTitle;
console.log(`Atualizando contagem de views para ${viewsText}`);
youtube.videos.update(
{
auth: auth,
part: 'snippet,statistics',
resource: video,
},
(err, response) => {
console.log(response)
if (err) {
console.log(`A contagem de views falhou ${err}`);
return;
}
if (response.data.items) {
console.log("Done");
}
}
);
}else{
console.log("Atualização não necessária");
};
}
})

This is a very broad question with too much to unpack. Keep in mind the differences below are very simplistically explained and there could be more steps.
Compute Engine/VM:
Step 1. Install javascript runtime (Nodejs) on the OS.
Step 2. Put your code & dependencies
Cloud Functions:
Step 1. Put your code & dependencies
Summary:
Cloud Functions removes the need to manage the operating system and
runtime. It might be easier for you. One way or another, you won't
escape the need for your dependency setup.
For Practice:
These "quick start" articles will give an understanding of the setup process. You learn by doing with a "hello world" example.
For Dependencies:
There is an article here that shows how to declare them in Cloud Functions. The sections of this article explains how you can load dependencies in different ways. The first example says: "The dependency is then imported in the function" and you see the part that says "require('escape-html')", it's the same as the "escape-html": "^1.0.3". This means that you can work backwards and see the requirements in your code to add them to your dependency manifest. But you may have to learn a bit more about the syntax by doing a Google search about package.json/npm and how it works in general.
For Authentication:
Assuming your code is properly written, the "require('googleapis')" at the top of your code means that you should only need to add dependencies like mentioned previously and your code may be good to go for authentication.
For Cron jobs:
In your code you are doing that by using "const cron = require("node-cron");" at the top of you code. This is like everything else I mentioned about dependencies and it's a software that triggers your code. I'm not sure if this will work like all other ones because Google has its own Cron jobs that work like this (outside the Cloud Functions environment, but triggers the Cloud Functions on the network). Keep in mind that this method could alter your code significantly. So if you can test with const cron = require("node-cron"); dependencies, then do that first.
Disclaimer:
Please open more specific questions if you can next time. Stackoverflow is not for consultation or broad questions with many follow-ups. It's more for coding answers to specific questions. Some may decide to close this post due to that.
Hope this helps.

How can I get a 'get' request to run on a schedule in NodeJS?

The function I would like this function to run by itself at time intervals. As it is now I have to visit the '/getCompanyInfo' path to trigger it. I would like it to run every minute as if I was visiting the '/getCompanyInfo' path each minute. The app is on Heroku and I would like the function to execute without any pages open.
The original function that is triggered by visiting the path.
const express = require('express');
const app = express();
/**
* getCompanyInfo ()
*/
app.get('/getCompanyInfo', function(req,res){
const companyID = oauthClient.getToken().realmId;
console.log(companyID)
const url = OAuthClient.environment.production ;
oauthClient.makeApiCall({url: url + 'v3/company/0000000000/salesreceipt/8?minorversion=41'})
.then(function(authResponse){
console.log("The response for API call is :"+JSON.parse(JSON.stringify(authResponse)));
res.send(authResponse);
})
.catch(function(e) {
console.error(e);
});
});
One of my attempts here was to put it in a function that executes each minute using node-schedule.
This one doesn't do anything other than print 'This will run once a minute.' to the console.
I tried removing
app.get(function(req,res){
and the
})
below it but that made the app (hosted on Heroku) fail to build.
const express = require('express');
const app = express();
var schedule = require('node-schedule');
var j = schedule.scheduleJob('* * * * *', function(){
console.log('This will run once a minute.');
app.get(function(req,res){
const companyID = oauthClient.getToken().realmId;
console.log(companyID)
const url = OAuthClient.environment.production ;
oauthClient.makeApiCall({url: url + 'v3/company/0000000000/salesreceipt/8?minorversion=41'})
.then(function(authResponse){
console.log("The response for API call is :"+JSON.parse(JSON.stringify(authResponse)));
res.send(authResponse);
})
.catch(function(e) {
console.error(e);
});
});
});
More Context:
It is inside an app I have on Heroku. I would like to set the app to make a requests for JSON data from the API every x time without me having to touch it.

app.get initializes api handler - e.g. this is your api route definition - the thing that will respond when you call GET /getCompanyInfo via web browser or some other client. You should not redefine it regularly with your scheduled action.
The failed build after you've removed the route handler is probably because of the res.send(authResponse); left behind.
You could have something like:
// function that will be used to get the data
const getCompanyInfo = (done) => {
const companyID = oauthClient.getToken().realmId
console.log(companyID)
const url = OAuthClient.environment.production
oauthClient.makeApiCall({url: url + 'v3/company/0000000000/salesreceipt/8?minorversion=41'})
.then((authResponse) => {
console.log("The response for API call is :"+JSON.parse(JSON.stringify(authResponse)))
done(authResponse)
})
.catch((e) => {
console.error(e)
})
}
// this will trigger the function regularly on the specified interval
const j = schedule.scheduleJob('* * * * *', () => {
getCompanyInfo((companyInfo) => {
// ...do whatever you want with the info
})
})
// this will return you the data by demand, when you call GET /getCompanyInfo via browser
app.get('/getCompanyInfo', function(req,res) {
getCompanyInfo((companyInfo) => {
res.send(companyInfo)
})
})

Heroku has an add on called Heroku Scheduler that does what you want. The node-schedule npm package might do the job, but as you mentioned, you probably aren't going to be able to see the execution/results/logs of your jobs that run every 24 hours without making some interface for it on your own.
For your issue, calling app.get doesn't make a lot of sense. That's just telling node about the route. Assuming you have your /getCompanyInfo route up and running, you just need to call it in your scheduled job, not re-register it every time.
You could also just do this (http being the http client you're using):
var j = schedule.scheduleJob('* * * * *', async function(){
console.log('This will run once a minute.');
const result = await http.get('/getCompanyInfo');
console.log(result);
});

Errors when using socket.io in AngularJs on user updates

Hi I'm trying to make an automated update on a list of articles whenever the user changes his preferred language.
The way I'm trying to do this is by having a IO socket update whenever the user changes in the database.
However I seem to be unsuccesfull in my endeavors, and I have no idea as to why.
Since I'm new to socket.io I thought I'll ask the coding gods in here for some help.
May the software be with you ^^
PS: the project is a Angular fullstack project, scaffolded with Yeoman
Code time!
client/ components/ articlebar/ articlebar.controller.js
'use strict';
angular.module('unityAcademyApp')
.controller('ArticlebarCtrl', function ($scope, $location, Auth, socket) {
$scope.articles = {};
function populateArticles(){
...
Some functionality where $scope.articles are set
...
};
socket.syncUpdates('user', $scope.articles, function() {
console.log('hit');
populateArticles();
});
});
client/ components/ socket/ socket.service.js
/* global io */
'use strict';
angular.module('unityAcademyApp')
.factory('socket', function(socketFactory) {
// socket.io now auto-configures its connection when we ommit a connection url
var ioSocket = io('', {
// Send auth token on connection, you will need to DI the Auth service above
// 'query': 'token=' + Auth.getToken()
path: '/socket.io-client'
});
var socket = socketFactory({
ioSocket: ioSocket
});
return {
socket: socket,
/**
* Register listeners to sync an array with updates on a model
*
* Takes the array we want to sync, the model name that socket updates are sent from,
* and an optional callback function after new items are updated.
*
* #param {String} modelName
* #param {Array} array
* #param {Function} cb
*/
syncUpdates: function (modelName, array, cb) {
cb = cb || angular.noop;
/**
* Syncs item creation/updates on 'model:save'
*/
socket.on(modelName + ':save', function (item) {
var oldItem = _.find(array, {_id: item._id});
var index = array.indexOf(oldItem); // this is line 39
var event = 'created';
// replace oldItem if it exists
// otherwise just add item to the collection
if (oldItem) {
array.splice(index, 1, item);
event = 'updated';
} else {
array.push(item);
}
cb(event, item, array);
});
/**
* Syncs removed items on 'model:remove'
*/
socket.on(modelName + ':remove', function (item) {
var event = 'deleted';
_.remove(array, {_id: item._id});
cb(event, item, array);
});
},
/**
* Removes listeners for a models updates on the socket
*
* #param modelName
*/
unsyncUpdates: function (modelName) {
socket.removeAllListeners(modelName + ':save');
socket.removeAllListeners(modelName + ':remove');
}
};
});
server/ config/ socketio.js
/**
* Socket.io configuration
*/
'use strict';
var config = require('./environment');
// When the user disconnects.. perform this
function onDisconnect(socket) {}
// When the user connects.. perform this
function onConnect(socket) {
// When the client emits 'info', this listens and executes
socket.on('info', function (data) {
console.info('[%s] %s', socket.address, JSON.stringify(data, null, 2));
});
// Insert sockets below
require('../api/translation/translation.socket').register(socket);
require('../api/comment/comment.socket').register(socket);
require('../api/article/article.socket').register(socket);
require('../api/language/language.socket').register(socket);
require('../api/thing/thing.socket').register(socket);
require('../api/user/user.socket').register(socket);
}
module.exports = function (socketio) {
// socket.io (v1.x.x) is powered by debug.
// In order to see all the debug output, set DEBUG (in server/config/local.env.js) to including the desired scope.
//
// ex: DEBUG: "http*,socket.io:socket"
// We can authenticate socket.io users and access their token through socket.handshake.decoded_token
//
// 1. You will need to send the token in `client/components/socket/socket.service.js`
//
// 2. Require authentication here:
// socketio.use(require('socketio-jwt').authorize({
// secret: config.secrets.session,
// handshake: true
// }));
socketio.on('connection', function (socket) {
socket.address = socket.handshake.address !== null ?
socket.handshake.address.address + ':' + socket.handshake.address.port :
process.env.DOMAIN;
socket.connectedAt = new Date();
// Call onDisconnect.
socket.on('disconnect', function () {
onDisconnect(socket);
console.info('[%s] DISCONNECTED', socket.address);
});
// Call onConnect.
onConnect(socket);
console.info('[%s] CONNECTED', socket.address);
});
};
server/ api/ user/ user.socket.js
/**
* Broadcast updates to client when the model changes
*/
'use strict';
var User = require('./user.model');
exports.register = function(socket) {
User.schema.post('save', function (doc) {
onSave(socket, doc);
});
User.schema.post('remove', function (doc) {
onRemove(socket, doc);
});
}
function onSave(socket, doc, cb) {
socket.emit('user:save', doc);
}
function onRemove(socket, doc, cb) {
socket.emit('user:remove', doc);
}
Errors encountered so far
So far I get the following error when running the code
TypeError: array.indexOf is not a function
at Socket.<anonymous> (socket.service.js:39)
at socket.js:24
at angular.js:17782
at completeOutstandingRequest (angular.js:5490)
at angular.js:5762
(anonymous function) # angular.js:12416
$get # angular.js:9203
(anonymous function) # angular.js:17785
completeOutstandingRequest # angular.js:5490
(anonymous function) # angular.js:5762

I'm not sure why you get that error, but I think I know why your data isn't updating.
You have to wrap your callback functions inside a $timeout function in order to trigger your changes. For example, you could do this:
$timeout(function(){
cb(event, item, array);
}, 0);
Remember to include $timeout directive in your socket factory.

what's 'undescore' mean? I'm not sure about the 'undescore', but I guess that's alias for 'this'. I think you should init var _ = this.
I just guessing.

I found the problem.
The error where due to it looking for a user in a list of articles, which returned undefined due to there not being any match. The solution was therefore to change the code in client/ components/ articlebar/ articlebar.controller.js
from
socket.syncUpdates('user', $scope.articles, function() {
console.log('hit');
populateArticles();
});
to
socket.syncUpdates('user', $scope.users, function() {
console.log('hit');
populateArticles();
});

Best approach for a modular node.js/socket.io/express application

I'm currectly creating an app using Node.JS that makes use of Express and Socket.io. As time progresses it's becoming increasingly difficult to deal with one file, I'm in the process of moving certain things out that I know how but was wondering on the best approach to do this.
I have a private area constructor similar to:
privateArea.js
function privateArea(props) {
this.id = props.id;
this.name = props.name;
this.users = [];
}
privateArea.prototype.addUser = function(socketId) {
this.users.push(socketId);
};
module.exports = privateArea;
I'd like to have this also have access to the socket.io variable that's been setup for use in a separate sockets.js file that can be included via the main app.js and a seperate file for express.js
So I'd like the structure as follows:
project
| app.js - joins it all together
| express.js - initialises and manages all express routing
| privateArea.js - constructor for private areas - must be able to reference socket.io
| sockets.js - initialises and manages all socket.io sockets and events
Any help/examples would be very appreciated.
Thanks

I use socket.io and express quite often in my projects, and I've developed a template which makes things easy. I like to have a fail-over in case the socket connections drops for some reason, or if a socket connection cannot be established. So I create http channels as well as socket channels. Here's a basic module template:
module.exports = function () {
var exported = {};
var someFunction = function (done) {
//.. code here..//
if (typeof done === "function") {
done(null, true);
}
};
// export the function
exported.someFunction = someFunction;
var apicalls = function (app) {
app.get("/module/someFunction", function (req, res) {
res.header("Content-Type", "application/json");
someFunction(function (err, response) {
if (err) return res.send(JSON.stringify(err));
res.send(JSON.stringify(response));
});
});
};
exported.apicalls = apicalls;
var socketcalls = function (io) {
io.on("connection", function (socket) {
socket.on('module-someFunction', function () {
someFunction(function (err, response) {
if (err) return socket.emit('module-someFunction', err);
socket.emit('module-someFunction', response);
});
});
});
};
exported.socketcalls = socketcalls;
return exported;
}
So to use this, I'd first need to include the module in my app.js file like this:
var mymod = require('./myModule.js');
And then I can enable access to this service from HTTP and over the websocket like this:
mymod.apicalls(app); // passing express to the module
mymod.socketcalls(io); // passing socket.io to the module
Finally, from the front-end, I can check to see if I have a socket connection, and if so, I use the socket to emit "module-someFunction". If I don't have a socket connection, the front-end will do an AJAX call instead to "/module/someFunction" which will hit the same function on the server side that it would've had I used the socket connection.
As an added bonus, if I need to utilize the function within the server, I could do that as well since the function is exported. That would look like this:
mymod.someFunction(function (err, response) {
// ... handle result here ... //
});

Creating a Mockup REST API

I'm currently trying to create a NodeJS server or something similar to mockup a REST API that reads in a JSON file and responds to requests with that data. I really only need GET requests supported. What is the best way to go about this?
Here's what I have so far:
/**
* Sample items REST API
*/
function ItemsRepository() {
this.items = [];
}
ItemsRepository.prototype.find = function (id) {
var item = this.items.filter(function(item) {
return item.itemId == id;
})[0];
if (null == item) {
throw new Error('item not found');
}
return item;
}
/**
* Retrieve all items
* items: array of items
*/
ItemsRepository.prototype.findAll = function () {
return this.items;
}
/**
* API
*/
var express = require('express');
var app = express();
var itemRepository = new ItemsRepository();
app.configure(function () {
// used to parse JSON object given in the body request
app.use(express.bodyParser());
});
/**
* HTTP GET /items
* items: the list of items in JSON format
*/
app.get('/items', function (request, response) {
response.json({items: itemRepository.findAll()});
});
/**
* HTTP GET /items/:id
* Param: :id is the unique identifier of the item you want to retrieve
* items: the item with the specified :id in a JSON format
* Error: 404 HTTP code if the item doesn't exists
*/
app.get('/items/:id', function (request, response) {
var itemId = request.params.id;
try {
response.json(itemRepository.find(itemId));
} catch (exception) {
response.send(404);
}
});
app.listen(8080); //to port on which the express server listen
I know that I would use the following to include the file, I just don't know how to stuff the data into Items.
var responseItemsData = require('./items-list.json');

This is trivial in node. You can load the data by requiring the .json file directly
var responseData = require('./my-json-file'); //.json extension optional
//Do this during your startup code, not during the request handler
Then to send it:
res.write(JSON.stringify(responseData));
The rest of the code you need is easily available in almost every node.js tutorial on the web.

You can use jasmine+sinon:
var Episode = Backbone.Model.extend({
url: function() {
return "/episode/" + this.id;
}
});
beforeEach(function() {
this.server = sinon.fakeServer.create();
});
afterEach(function() {
this.server.restore();
});
it("should fire the change event", function() {
var callback = sinon.spy();
this.server.respondWith("GET", "/episode/123",
[200, {"Content-Type": "application/json"},'{"id":123,"title":"Hollywood - Part 2"}']);
var episode = new Episode({id: 123});
// Bind to the change event on the model
episode.bind('change', callback);
// makes an ajax request to the server
episode.fetch();
// Fake server responds to the request
this.server.respond();
// Expect that the spy was called with the new model
expect(callback.called).toBeTruthy();
expect(callback.getCall(0).args[0].attributes)
.toEqual({id: 123,
title: "Hollywood - Part 2"});
});
more details in: https://github.com/cld-santos/simplologia/tree/master/javascript-lessons/src/test/javascript/Sinon

The easiest way is to simply use the static middleware.
var express = require('express');
var app = express();
app.use('/api', express.static(__dirname + '/data'));
app.use('.*', express.static(__dirname + '/assets'));
This assumes that you are eventually going to put the REST api at /api but that while you're testing your data is going to be in the data directory and that your CSS/JS/HTML is in the assets folder. In reality you can put it whereever you want, but you now can put all your dev json separate from your code.

I created a tool for this purpose
https://github.com/homerquan/kakuen