I have a website with multiple pages (http://www.europarl.europa.eu/sides/getDoc.do?type=PV&reference=20190131&secondRef=TOC&language=EN)
I crawled each page and got links with minutes in it. Each link with minutes has a date, after I created folders with Years and Months, I stuck with a store files from links.
The question is:
How can I download links in a month directory?
function crawlLink(link){
link = 'http://www.europarl.europa.eu'+link;
request(link, (error,response,
html) => {
if (!error && response.statusCode == 200) {
const $ = cheerio.load(html);
const docTitle = $('.doc_title' ) ;
var str = docTitle.html();
var date_str = str.replace(' - Brussels','').replace(' - Strasbourg', '');
var date = new Date(date_str);
console.log("created new dateobj", date);
var year = new Array('January','February', 'March' , 'April' , 'May' , 'June', 'July' ,'August','September','Oktober','November','December');
var mm = date.getMonth(); //January is 0
var yyyy = date.getFullYear();
var monthName = year[mm];
var yearDir = './data/'+yyyy;
if (!fs.existsSync(yearDir)){
fs.mkdirSync(yearDir);
}else{
console.log('yearDir exists');
}
var monthDir = yearDir+'/'+monthName;
if (!fs.existsSync(monthDir)){
fs.mkdirSync(monthDir);
}else{
console.log('monthDir exists');
}
console.log("wouhu everything is fine, get links and download them to monthDir");
let downloadLinks = [];
let $links = $('.doc_formats_box a');
$links.each(function(i, elem) {
downloadLinks.push({
title:$(this).text(),
link:$(this).attr('href')
});
});
console.log(downloadLinks);
`
Related
I have been working on a personal project of user check In/Out time recorder. I created the multi user login and time recorder using JS. I faced an issue to record the time of each logged in user. The code I created is only able to record the time of single user. I used localStorage command to store time because it need to be displayed every users time data on admin page. I am using single JS file for login, user page recorder and also for admin page loader. Plz help me out if anyone has any solution. I can't use server based code right now. I only have to write this web-app code in html,css and javascript.
var users= [
{user : 'user1', pass: 'pass1'},
{user : 'user2', pass: 'pass2'},
{user : 'user3', pass: 'pass3'},
{user : 'admin', pass: 'admin'}
];
function login()
{
//login page
var username = document.getElementById('uname').value;
var password = document.getElementById('psw').value;
for (var i = 0; i < users.length; i++)
{
console.log('enter the loop');
console.log("current user: ", i+1);
if (username == users[i].user && password == users[i].pass)
{
window.location.href = 'user.html';
alert("Login Successful");
break;
}
else if (i == (users.length-1))
{
alert('Login failed! Try Again');
console.log(i,'-times wrong');
}
else if (username == users[3].user && password == users[3].pass)
{
console.log('admin login');
window.location.href = 'admin.html';
break;
}
else
{
console.log('else statement');
}
}
var userId = i;
console.log("current user: ", userId);
}
//Date & Time Display
setInterval(myTimer,1000);
function myTimer()
{
const d = new Date();
document.getElementById('date').innerHTML = d.toLocaleDateString();
document.getElementById('time').innerHTML = d.toLocaleTimeString();
}
let date = document.getElementById('date');
let time = document.getElementById('time');
let newDate = new Date();
let year = newDate.getFullYear();
let month = newDate.getMonth();
let todaysDate = newDate.getDate();
let hours = newDate.getHours();
let minutes = newDate.getMinutes();
let seconds = newDate.getSeconds();
var cal = {
data : null,
sMth : 0,
sYear : 0,
init : () => {
let newDate = new Date();
cal.sDay = newDate.getDate();
cal.sMth = newDate.getMonth();
cal.hour = newDate.getHours();
cal.minutes = newDate.getMinutes();
cal.date = document.getElementById('date');
cal.data = localStorage.getItem("cal-" + cal.sDay + "-" + cal.sMth);
if (cal.data == null)
{
localStorage.setItem("cal-" + cal.sDay + "-" + cal.sMth, "{}");
cal.data = {};
console.log("cal.data: null-",cal.data);
}
else
{
cal.data = JSON.parse(cal.data);
console.log("cal.data: JSON-",cal.data);
}
},
checkIn : () => {
cal.data[cal.sDay] = cal.hour + ":" + cal.minutes;
localStorage.setItem('cal-${cal.sDay}-${cal.sMth}',JSON.stringify(cal.data));
console.log('Time:', cal.data);
}
}
window.onload = cal.init;
Reference to the previous question to Filter Gmail body and paste in Spreadsheet
I had created below a google app script and set a trigger for after every 5 minutes:
function GetEmailsData(){
// SKIP TO OUT OF OFFICE HOURS AND DAYS
var nowH=new Date().getHours();
var nowD=new Date().getDay();
if (nowH>19||nowH<8||nowD==0) { return }
// START OPERATION
var Gmail = GmailApp;
var ss = SpreadsheetApp.getActiveSpreadsheet();
var sheet = ss.getActiveSheet();
var MasterSheet = ss.getSheetByName("Master");
var index = 2;
var aa = 0; // count already update entries
var na = 0; // count not update entries
var lasttime = MasterSheet.getRange("Z1").getValue(); // in spreadsheet i record the last time of email scanned, so next trigger will search after that
Logger.log(lasttime);
var cdate = new Date();
var ctime = cdate.getTime();
var qDate = MasterSheet.getRange("Z1").getValue();
Logger.log("QDATE IS " + qDate);
//problem # 1: from filter is not working.
// SEARCH EMAIL
var query = 'from: email id, subject: email subject, after:' + Math.floor((qDate.getTime()) /1000);
var threadsNew = Gmail.search(query);
Logger.log(threadsNew.length);
//loop all emails
for(var n in threadsNew){
var thdNew = threadsNew[n];
var msgsNew = thdNew.getMessages();
var msgNew = msgsNew[msgsNew.length-1];
// GET ATTACHMENT
//var bodyNew = msgNew.getBody();
var plainbody = msgNew.getPlainBody();
var subject = msgNew.getSubject();
var Etime = msgNew.getDate();
//var attachments = msgNew.getAttachments();
//var attachment = attachments[0];
Logger.log(Etime);
Logger.log(subject);
//Logger.log(plainbody);
var tdata = plainbody.trim();
var data = parseEmail001(tdata);
//Logger.log(data);
// First Check Email Date
var newdate = new Date();
var dd = newdate.getDate();
var mm = newdate.getMonth() + 1;
var yyyy = newdate.getFullYear();
var cd = dd + "-" + mm + "-" + yyyy
//Logger.log(new Date());
//Logger.log(cd);
if (Etime.getDate() != dd) {return}
// first check current sheet exist or not
// DATE DECLARATION IS ABOVE
var itt = ss.getSheetByName(cd);
if (!itt) {
ss.insertSheet(cd);
var itt = ss.getSheetByName(cd);
var ms = ss.getSheetByName("Master");
var hlc = ms.getLastColumn();
var headings = ms.getRange(1,1,1,hlc).getValues();
itt.getRange(1,1,1,hlc).setValues(headings);
}
var MasterSheet = ss.getSheetByName(cd);
var mlr = MasterSheet.getLastRow();
var mlc = MasterSheet.getLastColumn();
//check data already updated or not
var NewDepositSlipNumber = Number(data[2]).toFixed(0);
//Logger.log(Number(data[2]).toFixed(0));
var olddata = MasterSheet.getDataRange().getValues();
//Logger.log(olddata[1][2]);
for(var i = 0; i<olddata.length;i++){
if(olddata[i][2] == NewDepositSlipNumber){
//Logger.log(i + 1);
var status = 'Already Update'
Logger.log(status);
break;
}
else{
var status = 'Not Update'
//Logger.log(status);
}
}
// count how many updated and how many not.
if(status == 'Not Update'){
na = na + 1;
}
else{
aa = aa + 1;
}
if(status == 'Not Update'){
MasterSheet.appendRow(data);
Logger.log("Data Updated");
}
}
//problem # 2: if conversation length are long, it is not reaching there
var lastscantime = threadsNew[0].getLastMessageDate();
var master = ss.getSheetByName("Master");
master.getRange("Z1").setValue(lastscantime);
Logger.log(lastscantime);
master.getRange("Z2").setValue(new Date());
Logger.log(new Date());
Logger.log("Total " + threadsNew.length + " found, out of which " + aa + " are already updated, while " + na + " are updated.");
}
function parseEmail001(message) {
var replaces = [
'Branch Code',
'Branch Name',
'Slip No',
'BL Number',
'Type Of Payment',
'Customer Name',
'Payer Bank',
'Payer Bank Branch',
'Transaction Type',
'Amount',
'Posting Date',
'Value Date'
];
return message.split('\n').slice(0, replaces.length)
.map((c,i) => c.replace(replaces[i], '').trim());
}
This script is working fine but I have two problems:
I placed three filters in query 1) from 2) subject 3) after
but sometimes it gives me emails that are from the same ID but different subject.
as per my observation, and I verified it by using debug mode if the conversation length is long I guess more than 30 it will not reach the end part of the script. it skipping my last step as I also marked it in the above script. and if conversation length is less it works smoothly.
explanation: in other words, from morning to evening it's working fine after every five minutes, but the next day it gets the problem, then I manually update my spreadsheet master sheet Z1 value as of today's date and time, then it works fine till tonight.
You might be reaching a script limitation. Possibly the custom function runtime.But it can be any, you should see an error if it didn't finish.
Objective is to show a link on the page based on a date
Since I am very new to JS, would there be a more efficient way to achieve this?
The below code works but ideally I would like to edit the HTML and not JS every time I want a change i.e enter dates and URL in HTML file and not the JS.
Links and dates are currently added manually in the JS
<!DOCTYPE html>
<html>
<body>
<p id="ademo"></p>
<script>
var yr = new Date().getFullYear();
var mn = new Date().getMonth()+1;
var dt = new Date().getDate();
var showlink;
var showlink1 = ('URL-1');
var showlink2 = ('URL-2');
var showlink3 = ('URL-3');
var showlink4 = ('URL-4');
if (yr == 2021 && mn == 3 && dt == 15) {
showlink = showlink1;
} else if (yr == 2021 && mn == 3 && dt == 16) {
showlink = showlink2;
} else if (yr == 2021 && mn == 3 && dt == 17) {
showlink = showlink3;
} else if (yr == 2021 && mn == 3 && dt == 18) {
showlink = showlink4;
}
document.getElementById("ademo").innerHTML = showlink.toString();
</script>
</body>
</html>
You could do it by adding a data object and manage that. This could also be an external JSON file you include in your code.
Adding your data into HTML will just bloat your HTML and likely have accessibility issues you need to deal with.
const data = {
'2021-3-15': 'URL-1',
'2021-3-16': 'URL-2',
'2021-3-17': 'URL-3',
'2021-3-18': 'URL-4',
};
const year = new Date().getFullYear();
const month = new Date().getMonth() + 1;
const day = new Date().getDate();
const key = `${year}-${month}-${day}`;
document.getElementById("ademo").innerHTML = data[key];
<p id="ademo"></p>
And if you want to us an external JSON file you'd do it this way:
async function loadData() {
const rawJson = await fetch('/url/to/your/data.json');
const data = await rawJson.json();
const year = new Date().getFullYear();
const month = new Date().getMonth() + 1;
const day = new Date().getDate();
const key = `${year}-${month}-${day}`;
document.getElementById("ademo").innerHTML = data[key];
}
loadData();
While your JSON would be in this shape:
{
"2021-3-15": "URL-1",
"2021-3-16": "URL-2",
"2021-3-17": "URL-3",
"2021-3-18": "URL-4"
}
For reference the full code (please ignore my attempts for scheduling)
var config = require("./config.js");
var Twit = require('twit');
var TT = new Twit(config);
///dt.setDate(25);
//console.log(dt)
var d = new Date();
const d1 = new Date('December 12, 2020 00:00:00');
const d2 = new Date('December 12, 2020 00:00:00');
const d3 = new Date('December 12, 2020 00:00:00');
//const dt = new Date('December 24, 2020 00:00:00');
d1.setDate(25);
d2.setDate(26);
d3.setDate(27);
tweetIt();
setInterval(tweetIt, 86400000)
function tweetIt(){
var calc = d1.getDate() - d.getDate();
if (d < d1.getDate()) {
var tweet = calc + " days left until Christmas! Ho Ho Ho!"
}
else if(d == d1.getDate() && d < d2.getDate()) {
var tweet = "Today is Christmas! Ho Ho Ho! "
}
else if(d == d2.getDate() && d < d3.getDate()){
var tweet = "Today is Christmas! Ho Ho Ho! "
}
else { console.log(tweet);}
TT.post('statuses/update', {status: tweet}, tweeted);
function tweeted(err, data, response) {
if (err) {
console.log(err);
} else {
console.log('Success: ' + data.text);
//console.log(response);
}
};
}
I tried different ways of getting the Twitter status to actually send something. One of the things I tried doing was to set var tweet = { status: "Hello!"} or just var tweet = "Hello 2 " (as it is in the code). Also when I do console.log(tweet) it shows undefined and I can't figure out why it isn't reading any of the declared parameters.
Also for another reference - the code before this one (currently the one running on my server, and works perfectly fine) / code 2
var config = require("./config.js");
var Twit = require('twit');
var T = new Twit(config);
tweetIt();
setInterval(tweetIt, 86400000)
function tweetIt(){
var d = new Date();
var calc = 25 - d.getDate();
var tweet = calc + " days left until Christmas! Ho Ho Ho! ❄⛄🎄";
T.post('statuses/update', { status: tweet }, tweeted);
function tweeted(err, data, response) {
if (err) {
console.log(err);
} else {
console.log('Success: ' + data.text);
//console.log(response);
}
};
}
I can't understand why code 1 doesn't work, because I used code 2 as a template for code 1. Help is highly appreciated!
Edit: Please, keep in mind that I'm a beginner!
I want to be able to set up HTML pages and load them into a single home page. Each html file will be named as the date (eg 03052016.html for today) and then the correct html will be pulled in each day on the homepage.
However not all days will have a html file, in which case it should roll back to a previous day. I have successfully loaded the page, nice and easy but can't work out a way to identify that the page hasn't loaded and subtract one from the day. My current attempt is the following:
<body>
<div id="success"></div>
<script>
//section creates the html file name for today
var today = new Date();
var dd = today.getDate();
var mm = today.getMonth()+1; //January is 0!
var yyyy = today.getFullYear();
if(dd<10) {
dd='0'+dd
}
if(mm<10) {
mm='0'+mm
}
today = dd+mm+yyyy+'.html';
var today = "05052016.html";
//do loop to subtract days until file is found
do{
var found = true; //variable records file presence
$( "#success" ).load( today, function( response, status, xhr ) {
if ( status == "error" ) {
var found = false;
if(parseInt(dd)>1){
dd = parseInt(dd)-1;
}else {
mm = parseInt(mm)-1;
dd = 30 //will deal with 31/30/28 day months later.
}
if(dd<10) {
dd='0'+dd
}
if(mm<10) {
mm='0'+mm
}
today = dd+mm+yyyy+'.html';
//
console.log( msg + xhr.status + " " + xhr.statusText );
}
});
}until(found == false )
</script>
I am new to web authoring so please be brutal if I am way off how to implement this. It seems so easy but the loop just won't work!
I am currently testing in FireFox, and using jquery-1.10.2.js
check de length of the content of the quuestioned div.
var div = $('div');
var dHtml = div.html();
var long = dHtml.length;
if(long > 0)
{
//do stuff
}
You need to understand that ajax (which is behind load) is asynchronous so you cannot test the found outside the success.
Try this:
function pad(num) {return String("0"+num).slice(-2)}
function getFilename(date)
var dd = pad(date.getDate());
var mm = pad(date.getMonth()+1); //January is 0!
var yyyy = date.getFullYear();
return ""+dd+mm+yyyy+".html";
}
var date = new Date(),
aDay = 10*24*60*60*1000,
giveUp = new Date(date.getTime()-(10*aDay)); // max 10 days back
function loadFile(date) {
$("#success").load( getFilename(date), function() {
if (status == "error") {
if (date>giveUp) {
date.setDate(date.getDate()-1)
loadFile(date);
}
}
});
}
$(function() {
loadFile(date);
});