JS: Overhaul import resolution

javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java

@@ -404,7 +404,7 @@ private void setupFilters() {
     patterns.add("**/*.view.json"); // SAP UI5
     patterns.add("**/manifest.json");
     patterns.add("**/package.json");
-    patterns.add("**/tsconfig*.json");
+    patterns.add("**/*tsconfig*.json");
     patterns.add("**/codeql-javascript-*.json");
 
     // include any explicitly specified extensions

javascript/extractor/src/com/semmle/js/extractor/JSONExtractor.java

@@ -29,7 +29,7 @@ public Context(Label parent, int childIndex) {
   private final boolean tolerateParseErrors;
 
   public JSONExtractor(ExtractorConfig config) {
-    this.tolerateParseErrors = config.isTolerateParseErrors();
+    this.tolerateParseErrors = true;
   }
 
   @Override

javascript/extractor/src/com/semmle/js/extractor/Main.java

@@ -301,9 +301,14 @@ public void setupMatchers(ArgsParser ap) {
     // only extract HTML and JS by default
     addIncludesFor(includes, FileType.HTML);
     addIncludesFor(includes, FileType.JS);
+    includes.add("**/.babelrc*.json");
+
 
     // extract TypeScript if `--typescript` or `--typescript-full` was specified
-    if (getTypeScriptMode(ap) != TypeScriptMode.NONE) addIncludesFor(includes, FileType.TYPESCRIPT);
+    if (getTypeScriptMode(ap) != TypeScriptMode.NONE) {
+      addIncludesFor(includes, FileType.TYPESCRIPT);
+      includes.add("**/*tsconfig*.json");
+    }
 
     // add explicit include patterns
     for (String pattern : ap.getZeroOrMore(P_INCLUDE))

javascript/ql/examples/snippets/importfrom.ql

@@ -11,5 +11,5 @@
 import javascript
 
 from ImportDeclaration id
-where id.getImportedPath().getValue() = "react"
+where id.getImportedPathString() = "react"
 select id

javascript/ql/lib/definitions.qll

@@ -70,7 +70,7 @@ private predicate importLookup(AstNode path, Module target, string kind) {
   kind = "I" and
   (
     exists(Import i |
-      path = i.getImportedPath() and
+      path = i.getImportedPathExpr() and
       target = i.getImportedModule()
     )
     or

javascript/ql/lib/semmle/javascript/AMD.qll

@@ -61,8 +61,14 @@ class AmdModuleDefinition extends CallExpr instanceof AmdModuleDefinition::Range
     result = this.getArgument(1)
   }
 
+  /** DEPRECATED. Use `getDependencyExpr` instead. */
+  deprecated PathExpr getDependency(int i) { result = this.getDependencyExpr(i) }
+
+  /** DEPRECATED. Use `getADependencyExpr` instead. */
+  deprecated PathExpr getADependency() { result = this.getADependencyExpr() }
+
   /** Gets the `i`th dependency of this module definition. */
-  PathExpr getDependency(int i) {
+  Expr getDependencyExpr(int i) {
     exists(Expr expr |
       expr = this.getDependencies().getElement(i) and
       not isPseudoDependency(expr.getStringValue()) and
@@ -71,8 +77,8 @@ class AmdModuleDefinition extends CallExpr instanceof AmdModuleDefinition::Range
   }
 
   /** Gets a dependency of this module definition. */
-  PathExpr getADependency() {
-    result = this.getDependency(_) or
+  Expr getADependencyExpr() {
+    result = this.getDependencyExpr(_) or
     result = this.getARequireCall().getAnArgument()
   }
 
@@ -233,7 +239,7 @@ private class AmdDependencyPath extends PathExprCandidate {
 }
 
 /** A constant path element appearing in an AMD dependency expression. */
-private class ConstantAmdDependencyPathElement extends PathExpr, ConstantString {
+deprecated private class ConstantAmdDependencyPathElement extends PathExpr, ConstantString {
   ConstantAmdDependencyPathElement() { this = any(AmdDependencyPath amd).getAPart() }
 
   override string getValue() { result = this.getStringValue() }
@@ -261,11 +267,13 @@ private predicate amdModuleTopLevel(AmdModuleDefinition def, TopLevel tl) {
  * An AMD dependency, viewed as an import.
  */
 private class AmdDependencyImport extends Import {
-  AmdDependencyImport() { this = any(AmdModuleDefinition def).getADependency() }
+  AmdDependencyImport() { this = any(AmdModuleDefinition def).getADependencyExpr() }
 
-  override Module getEnclosingModule() { this = result.(AmdModule).getDefine().getADependency() }
+  override Module getEnclosingModule() {
+    this = result.(AmdModule).getDefine().getADependencyExpr()
+  }
 
-  override PathExpr getImportedPath() { result = this }
+  override Expr getImportedPathExpr() { result = this }
 
   /**
    * Gets a file that looks like it might be the target of this import.
@@ -274,7 +282,7 @@ private class AmdDependencyImport extends Import {
    * adding well-known JavaScript file extensions like `.js`.
    */
   private File guessTarget() {
-    exists(PathString imported, string abspath, string dirname, string basename |
+    exists(FilePath imported, string abspath, string dirname, string basename |
       this.targetCandidate(result, abspath, imported, dirname, basename)
     |
       abspath.regexpMatch(".*/\\Q" + imported + "\\E")
@@ -296,9 +304,9 @@ private class AmdDependencyImport extends Import {
    * `dirname` and `basename` to the dirname and basename (respectively) of `imported`.
    */
   private predicate targetCandidate(
-    File f, string abspath, PathString imported, string dirname, string basename
+    File f, string abspath, FilePath imported, string dirname, string basename
   ) {
-    imported = this.getImportedPath().getValue() and
+    imported = this.getImportedPathString() and
     f.getStem() = imported.getStem() and
     f.getAbsolutePath() = abspath and
     dirname = imported.getDirName() and

javascript/ql/lib/semmle/javascript/ES2015Modules.qll

@@ -2,6 +2,7 @@
 
 import javascript
 private import semmle.javascript.internal.CachedStages
+private import semmle.javascript.internal.paths.PathExprResolver
 
 /**
  * An ECMAScript 2015 module.
@@ -91,7 +92,12 @@ private predicate hasDefaultExport(ES2015Module mod) {
 class ImportDeclaration extends Stmt, Import, @import_declaration {
   override ES2015Module getEnclosingModule() { result = this.getTopLevel() }
 
-  override PathExpr getImportedPath() { result = this.getChildExpr(-1) }
+  /**
+   * INTERNAL USE ONLY. DO NOT USE.
+   */
+  string getRawImportPath() { result = this.getChildExpr(-1).getStringValue() }
+
+  override Expr getImportedPathExpr() { result = this.getChildExpr(-1) }
 
   /**
    * Gets the object literal passed as part of the `with` (or `assert`) clause in this import declaration.
@@ -149,7 +155,7 @@ class ImportDeclaration extends Stmt, Import, @import_declaration {
 }
 
 /** A literal path expression appearing in an `import` declaration. */
-private class LiteralImportPath extends PathExpr, ConstantString {
+deprecated private class LiteralImportPath extends PathExpr, ConstantString {
   LiteralImportPath() { exists(ImportDeclaration req | this = req.getChildExpr(-1)) }
 
   override string getValue() { result = this.getStringValue() }
@@ -725,27 +731,12 @@ abstract class ReExportDeclaration extends ExportDeclaration {
   cached
   Module getReExportedModule() {
     Stages::Imports::ref() and
-    result.getFile() = this.getEnclosingModule().resolve(this.getImportedPath())
-    or
-    result = this.resolveFromTypeRoot()
-  }
-
-  /**
-   * Gets a module in a `node_modules/@types/` folder that matches the imported module name.
-   */
-  private Module resolveFromTypeRoot() {
-    result.getFile() =
-      min(TypeRootFolder typeRoot |
-        |
-        typeRoot.getModuleFile(this.getImportedPath().getStringValue())
-        order by
-          typeRoot.getSearchPriority(this.getFile().getParentContainer())
-      )
+    result.getFile() = ImportPathResolver::resolveExpr(this.getImportedPath())
   }
 }
 
 /** A literal path expression appearing in a re-export declaration. */
-private class LiteralReExportPath extends PathExpr, ConstantString {
+deprecated private class LiteralReExportPath extends PathExpr, ConstantString {
   LiteralReExportPath() { exists(ReExportDeclaration bred | this = bred.getImportedPath()) }
 
   override string getValue() { result = this.getStringValue() }

javascript/ql/lib/semmle/javascript/Expr.qll

@@ -2821,7 +2821,7 @@ class DynamicImportExpr extends @dynamic_import, Expr, Import {
     result = this.getSource().getFirstControlFlowNode()
   }
 
-  override PathExpr getImportedPath() { result = this.getSource() }
+  override Expr getImportedPathExpr() { result = this.getSource() }
 
   /**
    * Gets the second "argument" to the import expression, that is, the `Y` in `import(X, Y)`.
@@ -2852,7 +2852,7 @@ class DynamicImportExpr extends @dynamic_import, Expr, Import {
 }
 
 /** A literal path expression appearing in a dynamic import. */
-private class LiteralDynamicImportPath extends PathExpr, ConstantString {
+deprecated private class LiteralDynamicImportPath extends PathExpr, ConstantString {
   LiteralDynamicImportPath() {
     exists(DynamicImportExpr di | this.getParentExpr*() = di.getSource())
   }

javascript/ql/lib/semmle/javascript/Files.qll

@@ -29,6 +29,8 @@ private module Impl = Make<FsInput>;
 
 class Container = Impl::Container;
 
+module Folder = Impl::Folder;
+
 /** A folder. */
 class Folder extends Container, Impl::Folder {
   /** Gets the file or subfolder in this folder that has the given `name`, if any. */
@@ -73,6 +75,19 @@ class Folder extends Container, Impl::Folder {
       )
   }
 
+  /**
+   * Gets an implementation file and/or a typings file from this folder that has the given `stem`.
+   * This could be a single `.ts` file or a pair of `.js` and `.d.ts` files.
+   */
+  File getJavaScriptFileOrTypings(string stem) {
+    exists(File jsFile | jsFile = this.getJavaScriptFile(stem) |
+      result = jsFile
+      or
+      not jsFile.getFileType().isTypeScript() and
+      result = this.getFile(stem + ".d.ts")
+    )
+  }
+
   /** Gets a subfolder contained in this folder. */
   Folder getASubFolder() { result = this.getAChildContainer() }
 }

javascript/ql/lib/semmle/javascript/HTML.qll

@@ -214,7 +214,7 @@ module HTML {
         result = path.regexpCapture("file://(/.*)", 1)
         or
         not path.regexpMatch("(\\w+:)?//.*") and
-        result = this.getSourcePath().(ScriptSrcPath).resolve(this.getSearchRoot()).toString()
+        result = ResolveScriptSrc::resolve(this.getSearchRoot(), this.getSourcePath()).toString()
       )
     }
 
@@ -274,10 +274,16 @@ module HTML {
     )
   }
 
+  private module ResolverConfig implements Folder::ResolveSig {
+    predicate shouldResolve(Container base, string path) { scriptSrc(path, base) }
+  }
+
+  private module ResolveScriptSrc = Folder::Resolve<ResolverConfig>;
+
   /**
    * A path string arising from the `src` attribute of a `script` tag.
    */
-  private class ScriptSrcPath extends PathString {
+  deprecated private class ScriptSrcPath extends PathString {
     ScriptSrcPath() { scriptSrc(this, _) }
 
     override Folder getARootFolder() { scriptSrc(this, result) }

javascript/ql/lib/semmle/javascript/Modules.qll

@@ -6,6 +6,7 @@
 
 import javascript
 private import semmle.javascript.internal.CachedStages
+private import semmle.javascript.internal.paths.PathExprResolver
 
 /**
  * A module, which may either be an ECMAScript 2015-style module,
@@ -68,7 +69,7 @@ abstract class Module extends TopLevel {
    * This predicate is not part of the public API, it is only exposed to allow
    * overriding by subclasses.
    */
-  predicate searchRoot(PathExpr path, Folder searchRoot, int priority) {
+  deprecated predicate searchRoot(PathExpr path, Folder searchRoot, int priority) {
     path.getEnclosingModule() = this and
     priority = 0 and
     exists(string v | v = path.getValue() |
@@ -89,7 +90,7 @@ abstract class Module extends TopLevel {
    * resolves to a folder containing a main module (such as `index.js`), then
    * that file is the result.
    */
-  File resolve(PathExpr path) {
+  deprecated File resolve(PathExpr path) {
     path.getEnclosingModule() = this and
     (
       // handle the case where the import path is complete
@@ -122,8 +123,14 @@ abstract class Import extends AstNode {
   /** Gets the module in which this import appears. */
   abstract Module getEnclosingModule();
 
+  /** DEPRECATED. Use `getImportedPathExpr` instead. */
+  deprecated PathExpr getImportedPath() { result = this.getImportedPathExpr() }
+
   /** Gets the (unresolved) path that this import refers to. */
-  abstract PathExpr getImportedPath();
+  abstract Expr getImportedPathExpr();
+
+  /** Gets the imported path as a string. */
+  final string getImportedPathString() { result = this.getImportedPathExpr().getStringValue() }
 
   /**
    * Gets an externs module the path of this import resolves to.
@@ -132,45 +139,23 @@ abstract class Import extends AstNode {
    * path is assumed to be a possible target of the import.
    */
   Module resolveExternsImport() {
-    result.isExterns() and result.getName() = this.getImportedPath().getValue()
+    result.isExterns() and result.getName() = this.getImportedPathString()
   }
 
   /**
    * Gets the module the path of this import resolves to.
    */
-  Module resolveImportedPath() {
-    result.getFile() = this.getEnclosingModule().resolve(this.getImportedPath())
-  }
-
-  /**
-   * Gets a module with a `@providesModule` JSDoc tag that matches
-   * the imported path.
-   */
-  private Module resolveAsProvidedModule() {
-    exists(JSDocTag tag |
-      tag.getTitle() = "providesModule" and
-      tag.getParent().getComment().getTopLevel() = result and
-      tag.getDescription().trim() = this.getImportedPath().getValue()
-    )
-  }
+  Module resolveImportedPath() { result.getFile() = this.getImportedFile() }
 
   /**
-   * Gets a module in a `node_modules/@types/` folder that matches the imported module name.
+   * Gets the module the path of this import resolves to.
    */
-  private Module resolveFromTypeRoot() {
-    result.getFile() =
-      min(TypeRootFolder typeRoot |
-        |
-        typeRoot.getModuleFile(this.getImportedPath().getValue())
-        order by
-          typeRoot.getSearchPriority(this.getFile().getParentContainer())
-      )
-  }
+  File getImportedFile() { result = ImportPathResolver::resolveExpr(this.getImportedPathExpr()) }
 
   /**
-   * Gets the imported module, as determined by the TypeScript compiler, if any.
+   * DEPRECATED. Use `getImportedModule()` instead.
    */
-  private Module resolveFromTypeScriptSymbol() {
+  deprecated Module resolveFromTypeScriptSymbol() {
     exists(CanonicalName symbol |
       ast_node_symbol(this, symbol) and
       ast_node_symbol(result, symbol)
@@ -190,42 +175,11 @@ abstract class Import extends AstNode {
     Stages::Imports::ref() and
     if exists(this.resolveExternsImport())
     then result = this.resolveExternsImport()
-    else (
-      result = this.resolveAsProvidedModule() or
-      result = this.resolveImportedPath() or
-      result = this.resolveFromTypeRoot() or
-      result = this.resolveFromTypeScriptSymbol() or
-      result = resolveNeighbourPackage(this.getImportedPath().getValue())
-    )
+    else result = this.resolveImportedPath()
   }
 
   /**
    * Gets the data flow node that the default import of this import is available at.
    */
   abstract DataFlow::Node getImportedModuleNode();
 }
-
-/**
- * Gets a module imported from another package in the same repository.
- *
- * No support for importing from folders inside the other package.
- */
-private Module resolveNeighbourPackage(PathString importPath) {
-  exists(PackageJson json | importPath = json.getPackageName() and result = json.getMainModule())
-  or
-  exists(string package |
-    result.getFile().getParentContainer() = getPackageFolder(package) and
-    importPath = package + "/" + [result.getFile().getBaseName(), result.getFile().getStem()]
-  )
-}
-
-/**
- * Gets the folder for a package that has name `package` according to a package.json file in the resulting folder.
- */
-pragma[noinline]
-private Folder getPackageFolder(string package) {
-  exists(PackageJson json |
-    json.getPackageName() = package and
-    result = json.getFile().getParentContainer()
-  )
-}