I am working on a language that transpiles to javascript and has a similar syntax. However I want to include some new type of block statements. For syntax purposes they are the same as an IfStatement. How can I get esprima or acorn to parse this program MyStatement {a=1;} without throwing an error? Its fine if it calls it an IfStatement. I would prefer not to fork esprima.
It turns out, that the plugin capabilities of acorn are not really documented. It seems like forking acorn would be the easiest route. In this case, it is as simple as searching for occurances of _if and following a similar pattern for _MyStatement.
However it is possible to write a plugin to accomplish what I was trying to do. It seems a bit of a hack, but here is the code. The basic steps are:
To exend Parse and add to the list of keywords that will be recognized by the first pass
Create a TokenType for the new keyword and add it to the Parser.acorn.keywordTypes, extend parseStatement so that it processes the new TokenType
Create a handler for the new TokenType which will add information to the Abstract Syntax Tree as required by the keyword functionality and also consume tokens using commands like this.expect(tt.parenR) to eat a '(' or this.parseExpression() to process an entire expression.
Here is the code:
var program =
`
MyStatement {
MyStatement(true) {
MyStatement() {
var a = 1;
}
}
if (1) {
var c = 0;
}
}
`;
const acorn = require("acorn");
const Parser = acorn.Parser;
const tt = acorn.tokTypes; //used to access standard token types like "("
const TokenType = acorn.TokenType; //used to create new types of Tokens.
//add a new keyword to Acorn.
Parser.acorn.keywordTypes["MyStatement"] = new TokenType("MyStatement",{keyword: "MyStatement"});
//const isIdentifierStart = acorn.isIdentifierStart;
function wordsRegexp(words) {
return new RegExp("^(?:" + words.replace(/ /g, "|") + ")$")
}
var bruceware = function(Parser) {
return class extends Parser {
parse(program) {
console.log("hooking parse.");
//it appears it is necessary to add keywords here also.
var newKeywords = "break case catch continue debugger default do else finally for function if return switch throw try var while with null true false instanceof typeof void delete new in this const class extends export import super";
newKeywords += " MyStatement";
this.keywords = wordsRegexp(newKeywords);
return(super.parse(program));
}
parseStatement(context, topLevel, exports) {
var starttype = this.type;
console.log("!!!hooking parseStatement", starttype);
if (starttype == Parser.acorn.keywordTypes["MyStatement"]) {
console.log("Parse MyStatement");
var node = this.startNode();
return this.parseMyStatement(node);
}
else {
return(super.parseStatement(context, topLevel, exports));
}
}
parseMyStatement(node) {
console.log("parse MyStatement");
this.next();
//In my language, MyStatement doesn't have to have a parameter. It could be called as `MyStatement { ... }`
if (this.type == tt.parenL) {
node.test = this.parseOptionalParenExpression();
}
else {
node.test = 0; //If there is no test, just make it 0 for now (note that this may break code generation later).
}
node.isMyStatement = true; //set a flag so we know that this if a "MyStatement" instead of an if statement.
//process the body of the block just like a normal if statement for now.
// allow function declarations in branches, but only in non-strict mode
node.consequent = this.parseStatement("if");
//node.alternate = this.eat(acornTypes["else"]) ? this.parseStatement("if") : null;
return this.finishNode(node, "IfStatement")
};
//In my language, MyStatement, optionally has a parameter. It can also by called as MyStatement() { ... }
parseOptionalParenExpression() {
this.expect(tt.parenL);
//see what type it is
console.log("Type: ", this.type);
//allow it to be blank.
var val = 0; //for now just make the condition 0. Note that this may break code generation later.
if (this.type == tt.parenR) {
this.expect(tt.parenR);
}
else {
val = this.parseExpression();
this.expect(tt.parenR);
}
return val
};
}
}
process.stdout.write('\033c'); //cls
var result2 = Parser.extend(bruceware).parse(program); //attempt to parse
console.log(JSON.stringify(result2,null,' ')); //show the results.
I am writing unit tests for some of my functions and the tests runner seems to have a problem with a bound function. I am binding a function so I have reference to the this inside an inner function. Here is the code:
loadStates: function(name, stateName, options) {
if (myModule.getModule(name) !== undefined) {
this.prepState(name, stateName, options);
} else {
var bindForCheck = this.prepState.bind(this);
//module cannot be found check for 5 seconds
$log.warn("Requesting " + name + "...");
var timeToCheck = true;
setTimeout(function() {
timeToCheck = false;
}, 5000);
var check = {
init: function() {
check.checkAgain();
},
checkAgain: function() {
if (timeToCheck) {
if (myModule.getModule(name) !== undefined) {
bindForCheck(name, stateName, options);
} else {
//still doesn't exists
setTimeout(check.checkAgain, 200);
}
} else {
//doesn't exist after 5 seconds
$log.error("Requested module (" + name + ") could not be found at this time.");
}
}
};
check.init();
}
}
So the issue is with the
var bindForCheck = this.prepState.bind(this);
Which just lets me call an outer function inside of the check.checkAgain() function.
The test runner is spitting back this error when I try to run the else section of the function
TypeError: 'undefined' is not a function (evaluating 'this.prepState.bind(this)')
Could use some help here, this has me stumped on how to fix this. Thanks!
You are probably using PhantomJS of version < 2. Here is an issue on github. You gonna have to either update PhantomJS or use polyfill. There is a bower package that should do the trick.
It should be simple to create JavaScript intepreter in JavaScript using eval. I got this (using jQuery terminal):
term = $('#term_demo').terminal(function(command, term) {
if (command !== '') {
var result = window.eval("(" + command + ")");
if (result !== undefined) {
term.echo(String(result));
}
} else {
term.echo('');
}
}, {
greetings: 'Javascript Interpreter',
name: 'js_demo',
height: 200,
prompt: 'js> '
});
Demo
but it don't work when I execute function foo() { ... } the foo is not defined I need to use foo = function() { ... }. eval act like executed within (function() { <code> })(). Can it be there more complicated code that will not work too?
Is it possible to create a JavaScript interpreter using simple code without use the of js.js, that will work the same as browser console?
I've created a bookmarklet which appends a kind of REPL in a page, designed for the major five browsers (Chrome 1+, IE 6+, Firefox 1+, Safari 3+, Opera 9+Can't remember the exacte version).
The core component, which evaluates the code is posted below, slightly modified + annotated.
/**
* Evaluates some code in the global scope.
* #param String code: Code to eval
* #return Stringified result, prefixed with 'E:' if error.
*/
function globalEval(/*string*/ code) {
var win = window, r, raw;
try {
if (win.execScript) { // eval in IE sucks, so use execScript instead
r = win.$_$_$globalEval$_$_$ = {c:code};
win.execScript('try{$_$_$globalEval$_$_$.r=eval($_$_$globalEval$_$_$.c);}catch(e){$_$_$globalEval$_$_$.e=e}');
// /*Optional clean-up:*/ delete win.$_$_$globalEval$_$_$;
if (r.e) throw r.e; // Error
raw = r.r;
} else {
raw = win.eval(code);
}
r = '' + raw; // Stringify in the try-block
// It is possible that an error is thrown
// for example, for the following code: ({toString:1})
} catch(err) {
r = (err + ''); // Convert error to string
// IE: If found, "[object" will be at index zero, which is falsy
if (!r.indexOf('[object')) r = err.message;
// r =
r = 'E:' + (raw=r);
} finally {
// raw = unmodified result (or Error instance)
// FOR THIS EXAMPLE, raw is not used, and string r is returned
return /*string*/ r;
}
}
I've implemented the functionality in a form, which contains several controls including an input+output textarea.
Note: The code is evaluated in the global context. And such, any variables in code will be leaked to the global scope. For an interpreter, you could use an iframe to create a new scope (and modify var win in my function).
var win = frames['name_of_frame'], ... // or
var win = frame_DOM_element.contentWindow, ...
The parenthesis you're appending around your incoming command produces illegal syntax. You should wrap it with an anonymous self-executing function instead.
Example: http://jsfiddle.net/bW6Fv/1/
var command = "function foo(x){return x+x;} alert(foo(10));";
window.eval("!function(){" + command + "}()");
EDIT
If you want your evaluated scripts to be available globally, you'll have to leave them unwrapped or explicitly assign values to the global window object.
Example: http://jsfiddle.net/bW6Fv/2/
var command = "window.foo = function(x){return x+x;}; alert(foo(10));";
window.eval("!function(){ " + command + "}();");
window.eval("alert(foo(20));");
command = "function bar(x){return x / 2;}; alert(bar(10));";
window.eval(command);
window.eval("alert(bar(20));");
I have the following function definitions in the same .js file (call it A.js)
function PParser() {
....
makeExpression = function (lexemes, index) {
return makeNumber(lexemes, index);
}
makeDeclaration = function(lexemes, index)
{
if (lexemes[index].TokenType != LALLPLexer.VAR) {
throw "Expected VAR at index " + index;
}
if (lexemes[index + 1].TokenType != LALLPLexer.ID) {
throw "Expected ID at index " + index + 1;
}
if (lexemes[index + 2].TokenType != LALLPLexer.ASSIGN) {
throw "Expected ASSIGN at index " + index + 2;
}
var expressionNodeResult = makeExpression(lexemes, index + 3);
...
when the "makeExpression" invocation is reached, I was expecting control flow to move to the function defined just above. However, instead, another function named "makeExpression" is called in a completely different .js file (B.js).
function Controller()
{
...
this.parseToStatement = function(statementText)
{
makeExpression = function(expressionNode)
{
return new IntLiteral(expressionNode.Content);
}
try {
statement = parser.parseStatement(new LALLPLexer().lex(statementText));
if (statement.NodeType == LALLPParser.DECLARATION) {
return new Declaration(statement.Id, makeExpression(statement.Expression));
}
}
catch (exception) {
statement = new UnknownStatement(statementText);
}
return statement;
}
}
I'm not sure why. Interestingly enough, the line "parseStatement" shown above is up the call chain from the "makeExpression" invocation. Is this correct javascript behavior and, if so, why should I expect this behavior? How can I get the intended behavior?
I'm not entirely sure about this, but I think js doesn't make variables local by standard, so all those functions might be attached globally (To the window variable) and hence may overwrite each other.
Try adding var to all of those definitions (or this.where appropriate). Maybe that will help.
Given a function, I'm trying to find out the names of the nested functions in it (only one level deep).
A simple regex against toString() worked until I started using functions with comments in them. It turns out that some browsers store parts of the raw source while others reconstruct the source from what's compiled; The output of toString() may contain the original code comments in some browsers. As an aside, here are my findings:
Test subject
function/*post-keyword*/fn/*post-name*/()/*post-parens*/{
/*inside*/
}
document.write(fn.toString());
Results
Browser post-keyword post-name post-parens inside
----------- ------------ --------- ----------- --------
Firefox No No No No
Safari No No No No
Chrome No No Yes Yes
IE Yes Yes Yes Yes
Opera Yes Yes Yes Yes
I'm looking for a cross-browser way of extracting the nested function names from a given function. The solution should be able to extract "fn1" and "fn2" out of the following function:
function someFn() {
/**
* Some comment
*/
function fn1() {
alert("/*This is not a comment, it's a string literal*/");
}
function // keyword
fn2 // name
(x, y) // arguments
{
/*
body
*/
}
var f = function () { // anonymous, ignore
};
}
The solution doesn't have to be pure regex.
Update: You can assume that we're always dealing with valid, properly nested code with all string literals, comments and blocks terminated properly. This is because I'm parsing a function that has already been compiled as a valid function.
Update2: If you're wondering about the motivation behind this: I'm working on a new JavaScript unit testing framework that's called jsUnity. There are several different formats in which you can write tests & test suites. One of them is a function:
function myTests() {
function setUp() {
}
function tearDown() {
}
function testSomething() {
}
function testSomethingElse() {
}
}
Since the functions are hidden inside a closure, there's no way for me invoke them from outside the function. I therefore convert the outer function to a string, extract the function names, append a "now run the given inner function" statement at the bottom and recompile it as a function with new Function(). If the test function have comments in them, it gets tricky to extract the function names and to avoid false positives. Hence I'm soliciting the help of the SO community...
Update3: I've come up with a new solution that doesn't require a lot of semantic fiddling with code. I use the original source itself to probe for first-level functions.
Cosmetic changes and bugfix
The regular expression must read \bfunction\b to avoid false positives!
Functions defined in blocks (e.g. in the bodies of loops) will be ignored if nested does not evaluate to true.
function tokenize(code) {
var code = code.split(/\\./).join(''),
regex = /\bfunction\b|\(|\)|\{|\}|\/\*|\*\/|\/\/|"|'|\n|\s+/mg,
tokens = [],
pos = 0;
for(var matches; matches = regex.exec(code); pos = regex.lastIndex) {
var match = matches[0],
matchStart = regex.lastIndex - match.length;
if(pos < matchStart)
tokens.push(code.substring(pos, matchStart));
tokens.push(match);
}
if(pos < code.length)
tokens.push(code.substring(pos));
return tokens;
}
var separators = {
'/*' : '*/',
'//' : '\n',
'"' : '"',
'\'' : '\''
};
function extractInnerFunctionNames(func, nested) {
var names = [],
tokens = tokenize(func.toString()),
level = 0;
for(var i = 0; i < tokens.length; ++i) {
var token = tokens[i];
switch(token) {
case '{':
++level;
break;
case '}':
--level;
break;
case '/*':
case '//':
case '"':
case '\'':
var sep = separators[token];
while(++i < tokens.length && tokens[i] !== sep);
break;
case 'function':
if(level === 1 || (nested && level)) {
while(++i < tokens.length) {
token = tokens[i];
if(token === '(')
break;
if(/^\s+$/.test(token))
continue;
if(token === '/*' || token === '//') {
var sep = separators[token];
while(++i < tokens.length && tokens[i] !== sep);
continue;
}
names.push(token);
break;
}
}
break;
}
}
return names;
}
The academically correct way to handle this would be creating a lexer and parser for a subset of Javascript (the function definition), generated by a formal grammar (see this link on the subject, for example).
Take a look at JS/CC, for a Javascript parser generator.
Other solutions are just regex hacks, that lead to unmaintainable/unreadable code and probably to hidden parsing errors in particular cases.
As a side note, I'm not sure to understand why you aren't specifying the list of unit test functions in your product in a different way (an array of functions?).
Would it matter if you defined your tests like:
var tests = {
test1: function (){
console.log( "test 1 ran" );
},
test2: function (){
console.log( "test 2 ran" );
},
test3: function (){
console.log( "test 3 ran" );
}
};
Then you could run them as easily as this:
for( var test in tests ){
tests[test]();
}
Which looks much more easier.
You can even carry the tests around in JSON that way.
I like what you're doing with jsUnity. And when I see something I like (and have enough free time ;)), I try to reimplement it in a way which better suits my needs (also known as 'not-invented-here' syndrome).
The result of my efforts is described in this article, the code can be found here.
Feel free to rip-out any parts you like - you can assume the code to be in the public domain.
The trick is to basically generate a probe function that will check if a given name is the name of a nested (first-level) function. The probe function uses the function body of the original function, prefixed with code to check the given name within the scope of the probe function. OK, this can be better explained with the actual code:
function splitFunction(fn) {
var tokens =
/^[\s\r\n]*function[\s\r\n]*([^\(\s\r\n]*?)[\s\r\n]*\([^\)\s\r\n]*\)[\s\r\n]*\{((?:[^}]*\}?)+)\}\s*$/
.exec(fn);
if (!tokens) {
throw "Invalid function.";
}
return {
name: tokens[1],
body: tokens[2]
};
}
var probeOutside = function () {
return eval(
"typeof $fn$ === \"function\""
.split("$fn$")
.join(arguments[0]));
};
function extractFunctions(fn) {
var fnParts = splitFunction(fn);
var probeInside = new Function(
splitFunction(probeOutside).body + fnParts.body);
var tokens;
var fns = [];
var tokenRe = /(\w+)/g;
while ((tokens = tokenRe.exec(fnParts.body))) {
var token = tokens[1];
try {
if (probeInside(token) && !probeOutside(token)) {
fns.push(token);
}
} catch (e) {
// ignore token
}
}
return fns;
}
Runs fine against the following on Firefox, IE, Safari, Opera and Chrome:
function testGlobalFn() {}
function testSuite() {
function testA() {
function testNested() {
}
}
// function testComment() {}
// function testGlobalFn() {}
function // comments
testB /* don't matter */
() // neither does whitespace
{
var s = "function testString() {}";
}
}
document.write(extractFunctions(testSuite));
// writes "testA,testB"
Edit by Christoph, with inline answers by Ates:
Some comments, questions and suggestions:
Is there a reason for checking
typeof $fn$ !== "undefined" && $fn$ instanceof Function
instead of using
typeof $fn$ === "function"
instanceof is less safe than using typeof because it will fail when passing objects between frame boundaries. I know that IE returns wrong typeof information for some built-in functions, but afaik instanceof will fail in these cases as well, so why the more complicated but less safe test?
[AG] There was absolutely no legitimate reason for it. I've changed it to the simpler "typeof === function" as you suggested.
How are you going to prevent the wrongful exclusion of functions for which a function with the same name exists in the outer scope, e.g.
function foo() {}
function TestSuite() {
function foo() {}
}
[AG] I have no idea. Can you think of anything. Which one is better do you think? (a) Wrongful exclusion of a function inside. (b) Wronfgul inclusion of a function outside.
I started to think that the ideal solution will be a combination of your solution and this probing approach; figure out the real function names that are inside the closure and then use probing to collect references to the actual functions (so that they can be directly called from outside).
It might be possible to modify your implementation so that the function's body only has to be eval()'ed once and not once per token, which is rather inefficient. I might try to see what I can come up with when I have some more free time today...
[AG] Note that the entire function body is not eval'd. It's only the bit that's inserted to the top of the body.
[CG] Your right - the function's body only gets parsed once during the creation of probeInside - you did some nice hacking, there ;). I have some free time today, so let's see what I can come up with...
A solution that uses your parsing method to extract the real function names could just use one eval to return an array of references to the actual functions:
return eval("[" + fnList + "]");
[CG] Here is with what I came up. An added bonus is that the outer function stays intact and thus may still act as closure around the inner functions. Just copy the code into a blank page and see if it works - no guarantees on bug-freelessness ;)
<pre><script>
var extractFunctions = (function() {
var level, names;
function tokenize(code) {
var code = code.split(/\\./).join(''),
regex = /\bfunction\b|\(|\)|\{|\}|\/\*|\*\/|\/\/|"|'|\n|\s+|\\/mg,
tokens = [],
pos = 0;
for(var matches; matches = regex.exec(code); pos = regex.lastIndex) {
var match = matches[0],
matchStart = regex.lastIndex - match.length;
if(pos < matchStart)
tokens.push(code.substring(pos, matchStart));
tokens.push(match);
}
if(pos < code.length)
tokens.push(code.substring(pos));
return tokens;
}
function parse(tokens, callback) {
for(var i = 0; i < tokens.length; ++i) {
var j = callback(tokens[i], tokens, i);
if(j === false) break;
else if(typeof j === 'number') i = j;
}
}
function skip(tokens, idx, limiter, escapes) {
while(++idx < tokens.length && tokens[idx] !== limiter)
if(escapes && tokens[idx] === '\\') ++idx;
return idx;
}
function removeDeclaration(token, tokens, idx) {
switch(token) {
case '/*':
return skip(tokens, idx, '*/');
case '//':
return skip(tokens, idx, '\n');
case ')':
tokens.splice(0, idx + 1);
return false;
}
}
function extractTopLevelFunctionNames(token, tokens, idx) {
switch(token) {
case '{':
++level;
return;
case '}':
--level;
return;
case '/*':
return skip(tokens, idx, '*/');
case '//':
return skip(tokens, idx, '\n');
case '"':
case '\'':
return skip(tokens, idx, token, true);
case 'function':
if(level === 1) {
while(++idx < tokens.length) {
token = tokens[idx];
if(token === '(')
return idx;
if(/^\s+$/.test(token))
continue;
if(token === '/*') {
idx = skip(tokens, idx, '*/');
continue;
}
if(token === '//') {
idx = skip(tokens, idx, '\n');
continue;
}
names.push(token);
return idx;
}
}
return;
}
}
function getTopLevelFunctionRefs(func) {
var tokens = tokenize(func.toString());
parse(tokens, removeDeclaration);
names = [], level = 0;
parse(tokens, extractTopLevelFunctionNames);
var code = tokens.join('') + '\nthis._refs = [' +
names.join(',') + '];';
return (new (new Function(code)))._refs;
}
return getTopLevelFunctionRefs;
})();
function testSuite() {
function testA() {
function testNested() {
}
}
// function testComment() {}
// function testGlobalFn() {}
function // comments
testB /* don't matter */
() // neither does whitespace
{
var s = "function testString() {}";
}
}
document.writeln(extractFunctions(testSuite).join('\n---\n'));
</script></pre>
Not as elegant as LISP-macros, but still nice what JAvaScript is capable of ;)
<pre>
<script type="text/javascript">
function someFn() {
/**
* Some comment
*/
function fn1() {
alert("/*This is not a comment, it's a string literal*/");
}
function // keyword
fn2 // name
(x, y) // arguments
{
/*
body
*/
}
function fn3() {
alert("this is the word function in a string literal");
}
var f = function () { // anonymous, ignore
};
}
var s = someFn.toString();
// remove inline comments
s = s.replace(/\/\/.*/g, "");
// compact all whitespace to a single space
s = s.replace(/\s{2,}/g, " ");
// remove all block comments, including those in string literals
s = s.replace(/\/\*.*?\*\//g, "");
document.writeln(s);
// remove string literals to avoid false matches with the keyword 'function'
s = s.replace(/'.*?'/g, "");
s = s.replace(/".*?"/g, "");
document.writeln(s);
// find all the function definitions
var matches = s.match(/function(.*?)\(/g);
for (var ii = 1; ii < matches.length; ++ii) {
// extract the function name
var funcName = matches[ii].replace(/function(.+)\(/, "$1");
// remove any remaining leading or trailing whitespace
funcName = funcName.replace(/\s+$|^\s+/g, "");
if (funcName === '') {
// anonymous function, discard
continue;
}
// output the results
document.writeln('[' + funcName + ']');
}
</script>
</pre>
I'm sure I missed something, but from your requirements in the original question, I think I've met the goal, including getting rid of the possibility of finding the function keyword in string literals.
One last point, I don't see any problem with mangling the string literals in the function blocks. Your requirement was to find the function names, so I didn't bother trying to preserve the function content.