Feature/missing deps origin (#48)

* resolve missing deps origin

* change nonExistent from array to map in order to support specifying origin of missing dependency

* added support for yarn workspaces

* add flag to load package.json and not throw exception if it dosent exsit

* package.json fix

* resolve package dependencies to files

Author: amitgilad3

package.json

@@ -7,7 +7,7 @@
     "test": "jest",
     "test-watch": "jest --watch",
     "clean": "rm -rf dist",
-    "build": "babel src -d dist",
+    "build": "babel src -d dist  --source-maps",
     "watch": "babel --watch src -d dist --source-maps",
     "prepublishOnly": "npm run clean && npm run build",
     "build-debug": "babel src -d dist --source-maps"
@@ -49,6 +49,7 @@
     "is-relative-path": "^2.0.0",
     "lodash.flatten": "^4.4.0",
     "lodash.partition": "^4.6.0",
+    "lodash.set": "^4.3.2",
     "module-definition": "^2.2.4",
     "module-lookup-amd": "^5.0.1",
     "node-source-walk": "^3.3.0",

src/dependency-builder/build-tree.js

@@ -7,6 +7,7 @@ import R from 'ramda';
 import generateTree from './generate-tree-madge';
 import PackageJson from '../package-json/package-json';
 import partition from 'lodash.partition';
+import lset from 'lodash.set';
 import { DEFAULT_BINDINGS_PREFIX } from '../constants';
 
 /**
@@ -233,34 +234,42 @@ function groupMissing(missing, cwd, consumerPath, bindingPrefix) {
     if (item.startsWith(`${bindingPrefix}/`) || item.startsWith(`${DEFAULT_BINDINGS_PREFIX}/`)) return 'bits';
     return item.startsWith('.') ? 'files' : 'packages';
   });
-  const groups = byPathType(missing, bindingPrefix);
-  const packages = groups.packages ? groups.packages.map(resolvePackageNameByPath) : [];
+  const groups = Object.keys(missing).map(key => Object.assign({ originFile: path.relative(cwd, key) }, byPathType(missing[key], bindingPrefix)));
+  groups.forEach((group) => {
+    if (group.packages) group.packages = group.packages.map(resolvePackageNameByPath);
+  });
   // This is a hack to solve problems that madge has with packages for type script files
   // It see them as missing even if they are exists
   const foundPackages = {};
-  const missingPackages = [];
-  packages.forEach((packageName) => {
-    // Don't add the same package twice
-    if (R.contains(packageName, missingPackages)) return;
-    const resolvedPath = resolveModulePath(packageName, cwd, consumerPath);
-    if (!resolvedPath) {
-      return missingPackages.push(packageName);
-    }
-    const packageWithVersion = resolveNodePackage(cwd, resolvedPath);
+  const packageJson = PackageJson.findPackage(cwd);
+
+  groups.forEach((group) => {
+    const missingPackages = [];
+    if (group.packages) {
+      group.packages.forEach((packageName) => {
+      // Don't add the same package twice
+        if (R.contains(packageName, missingPackages)) return;
+        const resolvedPath = resolveModulePath(packageName, cwd, consumerPath);
+        if (!resolvedPath) {
+          return missingPackages.push(packageName);
+        }
+        const packageWithVersion = resolveNodePackage(cwd, resolvedPath);
 
-    return packageWithVersion ? Object.assign(foundPackages, packageWithVersion) :
-                                missingPackages.push(packageWithVersion);
+        return packageWithVersion ? Object.assign(foundPackages, packageWithVersion) :
+        missingPackages.push(packageWithVersion);
+      });
+    }
+    if (packageJson) {
+      const result = findPackagesInPackageJson(packageJson, missingPackages);
+      groups.packages = result.missingPackages;
+      Object.assign(foundPackages, result.foundPackages);
+    }
   });
-  groups.packages = missingPackages;
 
   // temporarily disable this functionality since it cause this bugs:
   // https://github.com/teambit/bit/issues/635
   // https://github.com/teambit/bit/issues/690
-  // if (packageJson) {
-  //   const result = findPackagesInPackageJson(packageJson, missingPackages);
-  //   groups.packages = result.missingPackages;
-  //   Object.assign(foundPackages, result.foundPackages)
-  // }
+
 
   return { groups, foundPackages };
 }
@@ -362,23 +371,23 @@ function updateTreeWithLinkFilesAndImportSpecifiers(tree: Tree, pathMap: PathMap
  * @param bindingPrefix
  * @return {Promise<{missing, tree}>}
  */
-export default async function getDependencyTree(baseDir: string, consumerPath: string, filePaths: string[], bindingPrefix: string):
-  Promise<{ missing: Object, tree: Tree}> {
+export default async function getDependencyTree(baseDir: string, consumerPath: string, filePaths: string[], bindingPrefix: string): Promise<{ missing: Object, tree: Tree}> {
   const config = { baseDir, includeNpm: true, requireConfig: null, webpackConfig: null, visited: {}, nonExistent: [] };
   const result = generateTree(filePaths, config);
   const { groups, foundPackages } = groupMissing(result.skipped, baseDir, consumerPath, bindingPrefix);
   const tree: Tree = groupDependencyTree(result.tree, baseDir, bindingPrefix);
-  const relativeFilePaths = filePaths.map(filePath => path.relative(baseDir, filePath));
+  // const relativeFilePaths = filePaths.map(filePath => path.relative(baseDir, filePath));
   // Merge manually found packages with madge founded packages
   if (foundPackages && !R.isEmpty(foundPackages)) {
     // Madge found packages so we need to merge them with the manual
-    relativeFilePaths.forEach((relativeFilePath) => {
-      if (tree[relativeFilePath].packages) {
-        Object.assign(tree[relativeFilePath].packages, foundPackages);
-        // There is only manually found packages
-      } else {
-        tree[relativeFilePath].packages = foundPackages;
-      }
+    Object.keys(foundPackages).forEach((pkg) => {
+      // locate package in groups(contains missing)
+      groups.forEach((fileDep) => {
+        if (fileDep.packages && fileDep.packages.includes(pkg)) {
+          fileDep.packages = fileDep.packages.filter(packageName => packageName !== pkg);
+          lset(tree[fileDep['originFile']],`packages.${pkg}`, foundPackages[pkg]);
+        }
+      });
     });
   }
 

src/dependency-builder/dependency-tree/index.js

@@ -35,7 +35,6 @@ module.exports = function(options) {
   var results = traverse(config);
   debug('traversal complete', results);
 
-  dedupeNonExistent(config.nonExistent);
   debug('deduped list of nonExistent partials: ', config.nonExistent);
 
   var tree;
@@ -100,13 +99,13 @@ module.exports._getDependencies = function(config) {
 
   var resolvedDependencies = [];
   var pathMapDependencies = [];
-  var pathMapFile = { file: config.filename };
+  var pathMapFile = {file: config.filename};
 
   for (var i = 0, l = dependencies.length; i < l; i++) {
     var dep = dependencies[i];
     const isVue = (!Array.isArray(dep) && dep.isScript);
     var result;
-    if (!Array.isArray(dep) && dep.isScript!== undefined) { // used for vue - return array of objects
+    if (!Array.isArray(dep) && dep.isScript !== undefined) { // used for vue - return array of objects
       result = cabinet({
         partial: dep.dep,
         filename: config.filename,
@@ -126,20 +125,29 @@ module.exports._getDependencies = function(config) {
         webpackConfig: config.webpackConfig
       });
     }
+    const dependency = isVue ? dep.dep : dep;
     if (!result) {
       debug('skipping an empty filepath resolution for partial: ' + dep);
-      config.nonExistent.push(isVue? dep.dep : dep);
+      if (config.nonExistent[config.filename]) {
+        config.nonExistent[config.filename].push(dependency);
+      } else {
+        config.nonExistent[config.filename] = [dependency];
+      }
       continue;
     }
 
     var exists = fs.existsSync(result);
 
     if (!exists) {
-      config.nonExistent.push(isVue? dep.dep : dep);
+      if (config.nonExistent[config.filename]) {
+        config.nonExistent[config.filename].push(dependency);
+      } else {
+        config.nonExistent[config.filename] = [dependency];
+      }
       debug('skipping non-empty but non-existent resolution: ' + result + ' for partial: ' + dep);
       continue;
     }
-    var pathMap = { dep: isVue ? dep.dep : dep , resolvedDep: result};
+    var pathMap = {dep: dependency , resolvedDep: result};
     var importSpecifiersSupportedLang = ['es6', 'ts', 'stylable'];
     importSpecifiersSupportedLang.forEach((lang) => {
       if (precinctOptions[lang] && precinctOptions[lang].importSpecifiers && precinctOptions[lang].importSpecifiers[dep]) {
@@ -232,12 +240,3 @@ function removeDups(list) {
   return unique;
 }
 
-// Mutate the list input to do a dereferenced modification of the user-supplied list
-function dedupeNonExistent(nonExistent) {
-  var deduped = removeDups(nonExistent);
-  nonExistent.length = deduped.length;
-
-  for (var i = 0, l = deduped.length; i < l; i++) {
-    nonExistent[i] = deduped[i];
-  }
-}

src/dependency-builder/generate-tree-madge.js

@@ -138,7 +138,7 @@ function addRelativePathsToPathMap(pathMap, pathCache, baseDir) {
 export default function generateTree(files, config) {
   const depTree = {};
   const visited = {};
-  const nonExistent = [];
+  const nonExistent = {};
   const npmPaths = {};
   const pathCache = {};
   const pathMap = [];