skip empty function tokens

Author: jakubzitny

tokenizers/javascript/index.js

@@ -63,8 +63,10 @@ const processFile = function(fileName, data) {
       const functionAstShallow = node
       const functionCode = regenerateFunctionCode(functionAstShallow)
       functions = functions.push(functionCode)
+
       const tokenizedFunction = tokenizer(functionCode, parentId, blockId++)
-      functionTokens = functionTokens.push(tokenizedFunction)
+      if (tokenizedFunction)
+        functionTokens = functionTokens.push(tokenizedFunction)
     }
   })
 

tokenizers/javascript/tokenizer.js

@@ -96,6 +96,9 @@ const tokenizer = function(code, parentId, blockId) {
     return `${token}${COUNT_DELIMITER}${count}`
   })
 
+  if (tokenPairs.size == 0)
+    return ''
+
   const lhs = `${parentId},${blockId},`
   const rhs = tokenPairs.join(TOKEN_DELIMITER)
   const output = `${lhs}${MAIN_DELIMITER}${rhs}`