diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 7704b4526..ef4cbdb1e 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -7,32 +7,34 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-22.04, windows-2022, macos-12] + os: [ubuntu-22.04, windows-2022, macos-14] steps: - name: Checkout github repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 1 - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: - distribution: 'temurin' - java-version: '11' - cache: 'sbt' + distribution: temurin + java-version: 11 + cache: sbt + - name: Set up SBT + uses: sbt/setup-sbt@v1 - name: Compile and run tests - run: sbt clean +test + run: sbt clean test formatting: runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 1 - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: - distribution: 'temurin' - java-version: '11' - cache: 'sbt' + distribution: temurin + java-version: 11 + cache: sbt - name: Check formatting run: sbt scalafmtCheck test:scalafmtCheck - run: echo "Previous step failed because code is not formatted. Run 'sbt scalafmt'" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 45046a197..2b9dec518 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -10,7 +10,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: Checkout github repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 - name: apt-get update @@ -22,13 +22,13 @@ jobs: env: PGP_SECRET: ${{ secrets.PGP_SECRET }} - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: - distribution: 'temurin' - java-version: '11' - cache: 'sbt' + distribution: temurin + java-version: 11 + cache: sbt - name: Run tests - run: sbt clean +test + run: sbt clean test - name: Release to sonatype run: sbt clean ciReleaseTagNextVersion ciReleaseSonatype env: diff --git a/README.md b/README.md index 9efbb625c..fd0231574 100644 --- a/README.md +++ b/README.md @@ -99,7 +99,7 @@ With this, we ensure to have ES6 compliant JS code before we continue with the a ## Parsing This is done by the [GraalJS Parser](https://github.com/oracle/graaljs/tree/master/graal-js/src/com.oracle.js.parser). -[Standard visitor pattern](https://github.com/ShiftLeftSecurity/js2cpg/blob/master/src/main/scala/io/shiftleft/js2cpg/parser/GeneralizingAstVisitor.scala) is used to traverse the resulting JS AST afterwards for our CPG-generation. +[Standard visitor pattern](https://github.com/ShiftLeftSecurity/js2cpg/blob/master/src/main/scala/io/shiftleft/js2cpg/parser/GeneralizingAstVisitor.scala) is used to traverse the resulting JS AST afterward for our CPG-generation. ## CPG-Generation diff --git a/build.sbt b/build.sbt index 567f6b9b6..f5beb5200 100644 --- a/build.sbt +++ b/build.sbt @@ -1,5 +1,5 @@ -val cpgVersion = "1.7.3" -val joernVersion = "4.0.8" +val cpgVersion = "1.7.37" +val joernVersion = "4.0.383" val gitCommitString = SettingKey[String]("gitSha") @@ -16,7 +16,7 @@ Global / excludeLintKeys += Fast / configuration Global / excludeLintKeys += gitCommitString lazy val commonSettings = Seq( - scalaVersion := "3.4.2", + scalaVersion := "3.6.4", organization := "io.shiftleft", scalacOptions ++= Seq("-Xtarget:8"), resolvers ++= Seq( @@ -28,14 +28,14 @@ lazy val commonSettings = Seq( "io.joern" %% "x2cpg" % joernVersion, "com.github.scopt" %% "scopt" % "4.1.0", "org.graalvm.js" % "js" % "22.3.4", - "com.fasterxml.jackson.core" % "jackson-databind" % "2.15.3", + "com.fasterxml.jackson.core" % "jackson-databind" % "2.19.1", "com.atlassian.sourcemap" % "sourcemap" % "2.0.0", - "commons-io" % "commons-io" % "2.13.0", + "commons-io" % "commons-io" % "2.19.0", "org.slf4j" % "slf4j-api" % "2.0.7", "org.apache.logging.log4j" % "log4j-slf4j2-impl" % "2.20.0" % Optional, "org.apache.logging.log4j" % "log4j-core" % "2.20.0" % Optional, "io.joern" %% "x2cpg" % joernVersion % Test classifier "tests", - "org.scalatest" %% "scalatest" % "3.2.18" % Test + "org.scalatest" %% "scalatest" % "3.2.19" % Test ) ) diff --git a/project/build.properties b/project/build.properties index ee4c672cd..bbb0b608c 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.10.1 +sbt.version=1.11.2 diff --git a/project/plugins.sbt b/project/plugins.sbt index fa69a01f3..f421ae16a 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,5 +1,6 @@ -addSbtPlugin("com.github.sbt" % "sbt-native-packager" % "1.9.16") +addSbtPlugin("com.github.sbt" % "sbt-native-packager" % "1.11.1") addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.5.0") -addSbtPlugin("io.shiftleft" % "sbt-ci-release-early" % "2.0.27") -addSbtPlugin("com.dwijnand" % "sbt-dynver" % "4.1.1") +addSbtPlugin("io.shiftleft" % "sbt-ci-release-early" % "2.1.7") +addSbtPlugin("com.github.sbt" % "sbt-dynver" % "5.1.0") addSbtPlugin("com.eed3si9n" % "sbt-buildinfo" % "0.10.0") +addSbtPlugin("com.github.sbt" % "sbt-git" % "2.1.0") diff --git a/src/main/scala/io/shiftleft/js2cpg/astcreation/AstCreator.scala b/src/main/scala/io/shiftleft/js2cpg/astcreation/AstCreator.scala index 5fa389faa..3f8e12a9a 100644 --- a/src/main/scala/io/shiftleft/js2cpg/astcreation/AstCreator.scala +++ b/src/main/scala/io/shiftleft/js2cpg/astcreation/AstCreator.scala @@ -1,74 +1,24 @@ package io.shiftleft.js2cpg.astcreation import com.oracle.js.parser.TokenType -import com.oracle.js.parser.ir.{ - AccessNode, - BinaryNode, - Block, - BlockExpression, - BlockStatement, - BreakNode, - CallNode, - CaseNode, - CatchNode, - ClassNode, - ContinueNode, - DebuggerNode, - ErrorNode, - Expression, - ExpressionStatement, - ForNode, - FunctionNode, - IdentNode, - IfNode, - ImportNode, - IndexNode, - JoinPredecessorExpression, - LabelNode, - LiteralNode, - Module, - Node, - ObjectNode, - ParameterNode, - PropertyNode, - ReturnNode, - Statement, - SwitchNode, - TemplateLiteralNode, - TernaryNode, - ThrowNode, - TryNode, - UnaryNode, - VarNode, - WhileNode, - WithNode -} +import com.oracle.js.parser.ir +import com.oracle.js.parser.ir.* import com.oracle.js.parser.ir.LiteralNode.ArrayLiteralNode import flatgraph.DiffGraphBuilder -import io.shiftleft.codepropertygraph.generated.nodes.{ - NewBlock, - NewCall, - NewControlStructure, - NewIdentifier, - NewLocal, - NewMethod, - NewMethodRef, - NewNamespaceBlock, - NewNode, - NewTypeDecl, - NewTypeRef -} -import io.shiftleft.codepropertygraph.generated.{ControlStructureTypes, DispatchTypes, ModifierTypes, Operators} -import io.shiftleft.js2cpg.datastructures.Stack._ -import io.shiftleft.js2cpg.datastructures._ -import io.shiftleft.js2cpg.datastructures.scope._ +import io.shiftleft.codepropertygraph.generated.nodes.* +import io.joern.x2cpg.datastructures.Stack.* +import io.joern.x2cpg.datastructures.VariableScopeManager +import io.joern.x2cpg.datastructures.VariableScopeManager.ScopeType +import io.joern.x2cpg.datastructures.VariableScopeManager.ScopeType.* +import io.shiftleft.codepropertygraph.generated.* +import io.shiftleft.js2cpg.datastructures.* import io.shiftleft.js2cpg.passes.{Defines, EcmaBuiltins, PassHelpers} import io.shiftleft.js2cpg.passes.PassHelpers.ParamNodeInitKind import io.shiftleft.js2cpg.parser.{GeneralizingAstVisitor, JsSource} import org.slf4j.LoggerFactory import scala.collection.mutable -import scala.jdk.CollectionConverters._ +import scala.jdk.CollectionConverters.* object AstCreator { @@ -83,20 +33,20 @@ object AstCreator { class AstCreator(diffGraph: DiffGraphBuilder, source: JsSource, usedIdentNodes: Set[String]) extends GeneralizingAstVisitor[NewNode] { - import AstCreator._ + import AstCreator.* - private val scope = new Scope() + private val scope = new VariableScopeManager() private val astEdgeBuilder = new AstEdgeBuilder(diffGraph) private val astNodeBuilder = new AstNodeBuilder(diffGraph, astEdgeBuilder, source, scope) // Nested methods are not put in the AST where they are defined. - // Instead we put them directly under the METHOD in which they are + // Instead, we put them directly under the METHOD in which they are // defined. To achieve this we need this extra stack. private val methodAstParentStack = new Stack[NewNode]() private val localAstParentStack = new Stack[NewBlock]() - private val switchExpressionStack = new Stack[Expression]() + private val switchExpressionStack = new Stack[ir.Expression]() private val dynamicInstanceTypeStack = mutable.Stack.empty[String] private val metaTypeRefIdStack = mutable.Stack.empty[NewTypeRef] @@ -141,11 +91,10 @@ class AstCreator(diffGraph: DiffGraphBuilder, source: JsSource, usedIdentNodes: methodAstParentStack.push(prepareFileWrapperFunction()) createImportsAndDependencies(programFunction.getModule) programFunction.accept(this) - createVariableReferenceLinks() + scope.createVariableReferenceLinks(diffGraph, source.filePath) } private def createImportsAndDependencies(module: Module): Unit = { - if (module == null) { return } @@ -195,16 +144,11 @@ class AstCreator(diffGraph: DiffGraphBuilder, source: JsSource, usedIdentNodes: private def createIdentifierNode(name: String, lineAndColumnProvider: Node): NewIdentifier = { val dynamicInstanceTypeOption = name match { - case "this" => - dynamicInstanceTypeStack.headOption - case "console" => - Some(Defines.Console) - case "Math" => - Some(Defines.Math) - case _ => - None + case "this" => dynamicInstanceTypeStack.headOption + case "console" => Some(Defines.Console) + case "Math" => Some(Defines.Math) + case _ => None } - astNodeBuilder.createIdentifierNode(name, lineAndColumnProvider, dynamicInstanceTypeOption) } @@ -230,11 +174,11 @@ class AstCreator(diffGraph: DiffGraphBuilder, source: JsSource, usedIdentNodes: val paramName = param.getName val localParamId = createIdentifierNode(paramName, param) val paramId = createIdentifierNode(name, param) - scope.addVariableReference(name, paramId) + scope.addVariableReference(name, paramId, Defines.Any, EvaluationStrategies.BY_REFERENCE) val localParamLocal = astNodeBuilder.createLocalNode(paramName, Defines.Any) addLocalToAst(localParamLocal) - scope.addVariable(paramName, localParamLocal, MethodScope) + scope.addVariable(paramName, localParamLocal, Defines.Any, MethodScope) val keyId = astNodeBuilder.createFieldIdentifierNode(paramName, param) val accessId = astNodeBuilder.createFieldAccessNode(paramId, keyId, astNodeBuilder.lineAndColumn(param)) val assignmentCallId = @@ -274,10 +218,10 @@ class AstCreator(diffGraph: DiffGraphBuilder, source: JsSource, usedIdentNodes: val localParamId = createIdentifierNode(paramName, parameter) val localParamLocal = astNodeBuilder.createLocalNode(paramName, Defines.Any) addLocalToAst(localParamLocal) - scope.addVariable(paramName, localParamLocal, MethodScope) + scope.addVariable(paramName, localParamLocal, Defines.Any, MethodScope) val paramId = createIdentifierNode(name, parameter) - scope.addVariableReference(name, paramId) + scope.addVariableReference(name, paramId, Defines.Any, EvaluationStrategies.BY_REFERENCE) val keyId = astNodeBuilder.createFieldIdentifierNode(paramName, parameter) @@ -454,7 +398,7 @@ class AstCreator(diffGraph: DiffGraphBuilder, source: JsSource, usedIdentNodes: // We do not need to look at classNode.getClassHeritage because // the CPG only allows us to encode inheriting from fully known // types. Since in JS we "inherit" from a variable which would - // need to be resolved first, we for now dont handle the class + // need to be resolved first, we for now don't handle the class // hierarchy. val astParentType = methodAstParentStack.head.label val astParentFullName = methodAstParentStack.head.propertiesMap.get("FULL_NAME").toString @@ -513,7 +457,7 @@ class AstCreator(diffGraph: DiffGraphBuilder, source: JsSource, usedIdentNodes: } if (classElement.isStatic) { - // Static member belong to the meta class. + // Static member belong to the metaclass. astEdgeBuilder.addAstEdge(memberId, metaTypeDeclId, memberOrderTracker) } else { astEdgeBuilder.addAstEdge(memberId, typeDeclId, memberOrderTracker) @@ -636,16 +580,15 @@ class AstCreator(diffGraph: DiffGraphBuilder, source: JsSource, usedIdentNodes: val receiverId = functionAccessNode.accept(this) val baseId = createIdentifierNode(baseIdentNode.getName, baseIdentNode) - scope.addVariableReference(baseIdentNode.getName, baseId) + scope.addVariableReference(baseIdentNode.getName, baseId, Defines.Any, EvaluationStrategies.BY_REFERENCE) (receiverId, None, receiverId, baseId) case base => // Base is complex so we need a tmp variable. - val tmpVarName = - PassHelpers.generateUnusedVariableName(usedVariableNames, usedIdentNodes, "_tmp") + val tmpVarName = PassHelpers.generateUnusedVariableName(usedVariableNames, usedIdentNodes, "_tmp") val baseTmpId = createIdentifierNode(tmpVarName, base) - scope.addVariableReference(tmpVarName, baseTmpId) + scope.addVariableReference(tmpVarName, baseTmpId, Defines.Any, EvaluationStrategies.BY_REFERENCE) val baseId = base.accept(this) @@ -666,16 +609,16 @@ class AstCreator(diffGraph: DiffGraphBuilder, source: JsSource, usedIdentNodes: ) val thisTmpId = createIdentifierNode(tmpVarName, functionAccessNode) - scope.addVariableReference(tmpVarName, thisTmpId) + scope.addVariableReference(tmpVarName, thisTmpId, Defines.Any, EvaluationStrategies.BY_REFERENCE) (baseId, Some(memberId), fieldAccessId, thisTmpId) } case _ => val receiverId = callNode.getFunction.accept(this) - // We need to create an synthetic this argument. + // We need to create a synthetic this argument. val thisId = createIdentifierNode("this", callNode) - scope.addVariableReference("this", thisId) + scope.addVariableReference("this", thisId, Defines.Any, EvaluationStrategies.BY_REFERENCE) (receiverId, None, receiverId, thisId) } @@ -976,7 +919,7 @@ class AstCreator(diffGraph: DiffGraphBuilder, source: JsSource, usedIdentNodes: expressionStatement.getExpression.accept(this) } - private def createRealBlock(block: Block): NewBlock = { + private def createRealBlock(block: ir.Block): NewBlock = { val blockId = astNodeBuilder.createBlockNode(block) val orderTracker = new OrderTracker() @@ -997,7 +940,7 @@ class AstCreator(diffGraph: DiffGraphBuilder, source: JsSource, usedIdentNodes: } // We do not get here for specially handled method top level blocks. - override def visit(block: Block): NewNode = { + override def visit(block: ir.Block): NewNode = { val realBlock = source.getString(block) == "{" if (realBlock) { @@ -1149,7 +1092,7 @@ class AstCreator(diffGraph: DiffGraphBuilder, source: JsSource, usedIdentNodes: override def visit(identNode: IdentNode): NewNode = { val identId = createIdentifierNode(identNode.getName, identNode) - scope.addVariableReference(identNode.getName, identId) + scope.addVariableReference(identNode.getName, identId, Defines.Any, EvaluationStrategies.BY_REFERENCE) identId } @@ -1220,12 +1163,12 @@ class AstCreator(diffGraph: DiffGraphBuilder, source: JsSource, usedIdentNodes: accessId } - private def createRhsForConditionalParameterInit(initExpression: Expression, varNode: VarNode): NewCall = { + private def createRhsForConditionalParameterInit(initExpression: ir.Expression, varNode: VarNode): NewCall = { createRhsForConditionalParameterInit(initExpression, varNode.getName.getName, varNode) } private def createRhsForConditionalParameterInit( - initExpression: Expression, + initExpression: ir.Expression, propertyNode: PropertyNode, name: Option[String] ): NewCall = { @@ -1237,18 +1180,19 @@ class AstCreator(diffGraph: DiffGraphBuilder, source: JsSource, usedIdentNodes: createRhsForConditionalParameterInit(initExpression, keyName, propertyNode) } - private def createRhsForConditionalParameterInit(initExpression: Expression, keyName: String, node: Node): NewCall = { + private def createRhsForConditionalParameterInit( + initExpression: ir.Expression, + keyName: String, + node: Node + ): NewCall = { val ternaryNode = initExpression.asInstanceOf[TernaryNode] val testId = { ternaryNode.getTest match { case binTestExpr: BinaryNode => val lhsId = createIdentifierNode(keyName, node) - scope.addVariableReference(keyName, lhsId) - - val rhsId = binTestExpr.getRhs.accept(this) - + scope.addVariableReference(keyName, lhsId, Defines.Any, EvaluationStrategies.BY_REFERENCE) + val rhsId = binTestExpr.getRhs.accept(this) val testCallId = astNodeBuilder.createEqualsCallNode(lhsId, rhsId, astNodeBuilder.lineAndColumn(binTestExpr)) - testCallId case otherExpr => otherExpr.accept(this) } @@ -1260,7 +1204,7 @@ class AstCreator(diffGraph: DiffGraphBuilder, source: JsSource, usedIdentNodes: ternaryNode.getFalseExpression.getExpression match { case paramNode: ParameterNode if paramNode.toString().startsWith("arguments") => val initId = createIdentifierNode(keyName, node) - scope.addVariableReference(keyName, initId) + scope.addVariableReference(keyName, initId, Defines.Any, EvaluationStrategies.BY_REFERENCE) initId case _ => ternaryNode.getFalseExpression.accept(this) } @@ -1274,7 +1218,7 @@ class AstCreator(diffGraph: DiffGraphBuilder, source: JsSource, usedIdentNodes: .foreach(id => astNodeBuilder.createDependencyNode(name, id, VERSION_REQUIRE)) } - private def createVarNodeParamNodeInitKindFalse(varNode: VarNode, assignmentSource: Expression): NewNode = { + private def createVarNodeParamNodeInitKindFalse(varNode: VarNode, assignmentSource: ir.Expression): NewNode = { val (typeFullName, code) = if (varNode.isFunctionDeclaration) { val functionNode = varNode.getInit.asInstanceOf[FunctionNode] @@ -1291,7 +1235,7 @@ class AstCreator(diffGraph: DiffGraphBuilder, source: JsSource, usedIdentNodes: } else { MethodScope } - scope.addVariable(varNode.getName.getName, varId, scopeType) + scope.addVariable(varNode.getName.getName, varId, Defines.Any, scopeType) if (varNode.isAssignment) { val destId = varNode.getAssignmentDest.accept(this) @@ -1876,93 +1820,12 @@ class AstCreator(diffGraph: DiffGraphBuilder, source: JsSource, usedIdentNodes: blockId } - private def createVariableReferenceLinks(): Unit = { - val resolvedReferenceIt = scope.resolve(createMethodLocalForUnresolvedReference) - val capturedLocals = mutable.HashMap.empty[String, NewNode] - - resolvedReferenceIt.foreach { case ResolvedReference(variableNodeId, origin) => - var currentScope = origin.stack - var currentReferenceId = origin.referenceNodeId - var nextReferenceId: NewNode = null - - var done = false - while (!done) { - val localOrCapturedLocalIdOption = - if (currentScope.get.nameToVariableNode.contains(origin.variableName)) { - done = true - Some(variableNodeId) - } else { - currentScope.flatMap { - case methodScope: MethodScopeElement => - // We have reached a MethodScope and still did not find a local variable to link to. - // For all non local references the CPG format does not allow us to link - // directly. Instead we need to create a fake local variable in method - // scope and link to this local which itself carries the information - // that it is a captured variable. This needs to be done for each - // method scope until we reach the originating scope. - val closureBindingIdProperty = - methodScope.methodFullName + ":" + origin.variableName - capturedLocals - .updateWith(closureBindingIdProperty) { - case None => - val methodScopeNodeId = methodScope.scopeNode - val localId = - astNodeBuilder.createLocalNode(origin.variableName, Defines.Any, Some(closureBindingIdProperty)) - astEdgeBuilder.addAstEdge(localId, methodScopeNodeId, 0) - val closureBindingId = - astNodeBuilder.createClosureBindingNode(closureBindingIdProperty, origin.variableName) - - methodScope.capturingRefId.foreach(astEdgeBuilder.addCaptureEdge(closureBindingId, _)) - - nextReferenceId = closureBindingId - - Some(localId) - case someLocalId => - // When there is already a LOCAL representing the capturing, we do not - // need to process the surrounding scope element as this has already - // been processed. - done = true - someLocalId - } - case _: BlockScopeElement => None - } - } - - localOrCapturedLocalIdOption.foreach { localOrCapturedLocalId => - astEdgeBuilder.addRefEdge(localOrCapturedLocalId, currentReferenceId) - currentReferenceId = nextReferenceId - } - - currentScope = currentScope.get.surroundingScope - } - } - } - - private def createMethodLocalForUnresolvedReference( - methodScopeNodeId: NewNode, - variableName: String - ): (NewNode, ScopeType) = { - val varId = - astNodeBuilder.createLocalNode(variableName, Defines.Any) - astEdgeBuilder.addAstEdge(varId, methodScopeNodeId, 0) - (varId, MethodScope) - } - - private def computeScopePath(stack: Option[ScopeElement]): String = - new ScopeElementIterator(stack) - .to(Seq) - .reverse - .collect { case methodScopeElement: MethodScopeElement => - methodScopeElement.name - } - .mkString(":") - private def calcTypeNameAndFullName(classNode: ClassNode): (String, String) = { def calcTypeName(classNode: ClassNode): String = { val typeName = Option(classNode.getIdent) match { case Some(ident) => ident.getName - // in JS it is possible to create anonymous classes; hence no name - case None => + case None => + // In JS, it is possible to create anonymous classes; hence no name "_anon_cdecl" } typeName @@ -1973,7 +1836,7 @@ class AstCreator(diffGraph: DiffGraphBuilder, source: JsSource, usedIdentNodes: nameAndFullName case None => val name = calcTypeName(classNode) - val fullNamePrefix = source.filePath + ":" + computeScopePath(scope.getScopeHead) + ":" + val fullNamePrefix = source.filePath + ":" + scope.computeScopePath + ":" val intendedFullName = fullNamePrefix + name val postfix = typeFullNameToPostfix.getOrElse(intendedFullName, 0) @@ -1994,18 +1857,12 @@ class AstCreator(diffGraph: DiffGraphBuilder, source: JsSource, usedIdentNodes: private def calcMethodNameAndFullName(functionNode: FunctionNode): (String, String) = { def calcMethodName(functionNode: FunctionNode): String = { val name = functionNode match { - case _ if functionNode.isAnonymous && functionNode.isProgram => - ":program" - case _ if functionNode.isAnonymous && functionNode.isClassConstructor => - "anonClass" - case _ if functionNode.isAnonymous => - "anonymous" - case _ if functionNode.isClassConstructor => - s"${functionNode.getName}" - case _ => - functionNode.getName + case _ if functionNode.isAnonymous && functionNode.isProgram => ":program" + case _ if functionNode.isAnonymous && functionNode.isClassConstructor => "anonClass" + case _ if functionNode.isAnonymous => "anonymous" + case _ if functionNode.isClassConstructor => s"${functionNode.getName}" + case _ => functionNode.getName } - name } @@ -2017,7 +1874,7 @@ class AstCreator(diffGraph: DiffGraphBuilder, source: JsSource, usedIdentNodes: nameAndFullName case None => val intendedName = calcMethodName(functionNode) - val fullNamePrefix = source.filePath + ":" + computeScopePath(scope.getScopeHead) + ":" + val fullNamePrefix = source.filePath + ":" + scope.computeScopePath + ":" var name = intendedName var fullName = "" diff --git a/src/main/scala/io/shiftleft/js2cpg/astcreation/AstEdgeBuilder.scala b/src/main/scala/io/shiftleft/js2cpg/astcreation/AstEdgeBuilder.scala index bb3668201..a7d1382b4 100644 --- a/src/main/scala/io/shiftleft/js2cpg/astcreation/AstEdgeBuilder.scala +++ b/src/main/scala/io/shiftleft/js2cpg/astcreation/AstEdgeBuilder.scala @@ -2,7 +2,7 @@ package io.shiftleft.js2cpg.astcreation import flatgraph.DiffGraphBuilder import io.shiftleft.codepropertygraph.generated.EdgeTypes -import io.shiftleft.codepropertygraph.generated.nodes._ +import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.js2cpg.datastructures.OrderTracker import org.slf4j.LoggerFactory @@ -82,10 +82,6 @@ class AstEdgeBuilder(private val diffGraph: DiffGraphBuilder) { diffGraph.addEdge(srcId, dstId, EdgeTypes.REF) } - def addCaptureEdge(dstId: NewNode, srcId: NewNode): Unit = { - diffGraph.addEdge(srcId, dstId, EdgeTypes.CAPTURE) - } - def addBindsEdge(dstId: NewNode, srcId: NewNode): Unit = { diffGraph.addEdge(srcId, dstId, EdgeTypes.BINDS) addOrder(srcId, 0) diff --git a/src/main/scala/io/shiftleft/js2cpg/astcreation/AstNodeBuilder.scala b/src/main/scala/io/shiftleft/js2cpg/astcreation/AstNodeBuilder.scala index 8bb89439c..8d7aef390 100644 --- a/src/main/scala/io/shiftleft/js2cpg/astcreation/AstNodeBuilder.scala +++ b/src/main/scala/io/shiftleft/js2cpg/astcreation/AstNodeBuilder.scala @@ -1,11 +1,12 @@ package io.shiftleft.js2cpg.astcreation -import com.oracle.js.parser.ir._ +import com.oracle.js.parser.ir.* import flatgraph.DiffGraphBuilder -import io.shiftleft.codepropertygraph.generated.nodes._ +import io.joern.x2cpg.datastructures.VariableScopeManager +import io.joern.x2cpg.datastructures.VariableScopeManager.ScopeType.* +import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.codepropertygraph.generated.{DispatchTypes, EvaluationStrategies, Operators} import io.shiftleft.js2cpg.datastructures.{LineAndColumn, OrderTracker} -import io.shiftleft.js2cpg.datastructures.scope.{MethodScope, Scope} import io.shiftleft.js2cpg.passes.Defines import io.shiftleft.js2cpg.parser.JsSource import io.shiftleft.js2cpg.parser.JsSource.shortenCode @@ -14,12 +15,9 @@ class AstNodeBuilder( private val diffGraph: DiffGraphBuilder, private val astEdgeBuilder: AstEdgeBuilder, private val source: JsSource, - private val scope: Scope + private val scope: VariableScopeManager ) { - implicit def int2IntegerOpt(x: Option[Int]): Option[Integer] = x.map(java.lang.Integer.valueOf) - implicit def int2Integer(x: Int): Integer = java.lang.Integer.valueOf(x) - def codeOf(node: NewNode): String = node match { case node: AstNodeNew => node.code case _ => "" @@ -67,7 +65,7 @@ class AstNodeBuilder( diffGraph.addNode(param) orderTracker.inc() astEdgeBuilder.addAstEdge(param, methodNode) - scope.addVariable(name, param, MethodScope) + scope.addVariable(name, param, Defines.Any, MethodScope) param } @@ -351,16 +349,6 @@ class AstNodeBuilder( namespaceBlock } - def createClosureBindingNode(closureBindingId: String, closureOriginalName: String): NewClosureBinding = { - val closureBinding = NewClosureBinding() - .closureBindingId(Some(closureBindingId)) - .evaluationStrategy(EvaluationStrategies.BY_REFERENCE) - .closureOriginalName(Some(closureOriginalName)) - - diffGraph.addNode(closureBinding) - closureBinding - } - def createMethodRefNode(code: String, methodFullName: String, functionNode: FunctionNode): NewMethodRef = { val lineColumn = lineAndColumn(functionNode) val line = lineColumn.line @@ -491,9 +479,8 @@ class AstNodeBuilder( } def createLocalNode(name: String, typeFullName: String, closureBindingId: Option[String] = None): NewLocal = { - val code = "N/A" val local = NewLocal() - .code(shortenCode(code)) + .code(name) .name(name) .typeFullName(typeFullName) .closureBindingId(closureBindingId) diff --git a/src/main/scala/io/shiftleft/js2cpg/astcreation/NewCompositeNode.scala b/src/main/scala/io/shiftleft/js2cpg/astcreation/NewCompositeNode.scala index 253754471..7a724afb3 100644 --- a/src/main/scala/io/shiftleft/js2cpg/astcreation/NewCompositeNode.scala +++ b/src/main/scala/io/shiftleft/js2cpg/astcreation/NewCompositeNode.scala @@ -32,6 +32,6 @@ class NewCompositeNode(underlying: ListBuffer[NewNode] = ListBuffer.empty[NewNod override def isValidInNeighbor(edgeLabel: String, node: NewNode): Boolean = ??? // we do not need this override def isValidOutNeighbor(edgeLabel: String, node: NewNode): Boolean = ??? // we do not need this override def propertiesMap: util.Map[String, Any] = ??? // we do not need this - override def countAndVisitProperties(interface: BatchedUpdateInterface): Unit = ??? + override def countAndVisitProperties(interface: BatchedUpdateInterface): Unit = ??? // we do not need this override type StoredNodeType = StoredNode } diff --git a/src/main/scala/io/shiftleft/js2cpg/core/Js2Cpg.scala b/src/main/scala/io/shiftleft/js2cpg/core/Js2Cpg.scala index d867d2b3c..e146221f7 100644 --- a/src/main/scala/io/shiftleft/js2cpg/core/Js2Cpg.scala +++ b/src/main/scala/io/shiftleft/js2cpg/core/Js2Cpg.scala @@ -3,15 +3,15 @@ package io.shiftleft.js2cpg.core import java.nio.file.{Path, StandardCopyOption} import better.files.File import better.files.File.LinkOptions -import io.shiftleft.js2cpg.passes._ -import io.shiftleft.js2cpg.io.FileDefaults._ +import io.shiftleft.js2cpg.passes.* +import io.shiftleft.js2cpg.io.FileDefaults.* import io.shiftleft.js2cpg.io.FileUtils import io.shiftleft.js2cpg.parser.PackageJsonParser import io.shiftleft.js2cpg.preprocessing.NuxtTranspiler import io.shiftleft.js2cpg.preprocessing.TranspilationRunner import io.shiftleft.js2cpg.utils.MemoryMetrics import io.joern.x2cpg.X2Cpg.newEmptyCpg -import io.joern.x2cpg.utils.HashUtil +import io.joern.x2cpg.utils.{HashUtil, Report} import org.slf4j.LoggerFactory import scala.util.{Failure, Success, Try} diff --git a/src/main/scala/io/shiftleft/js2cpg/core/Js2CpgMain.scala b/src/main/scala/io/shiftleft/js2cpg/core/Js2CpgMain.scala index 24f957d14..0d1531029 100644 --- a/src/main/scala/io/shiftleft/js2cpg/core/Js2CpgMain.scala +++ b/src/main/scala/io/shiftleft/js2cpg/core/Js2CpgMain.scala @@ -2,8 +2,7 @@ package io.shiftleft.js2cpg.core object Js2CpgMain { def main(args: Array[String]): Unit = { - val argumentsParser: Js2cpgArgumentsParser = new Js2cpgArgumentsParser() - + val argumentsParser = new Js2cpgArgumentsParser() argumentsParser.parse(args) match { case Some(config) => new Js2Cpg().run(config) diff --git a/src/main/scala/io/shiftleft/js2cpg/core/Js2cpgArgumentsParser.scala b/src/main/scala/io/shiftleft/js2cpg/core/Js2cpgArgumentsParser.scala index a685bec66..cc17840da 100644 --- a/src/main/scala/io/shiftleft/js2cpg/core/Js2cpgArgumentsParser.scala +++ b/src/main/scala/io/shiftleft/js2cpg/core/Js2cpgArgumentsParser.scala @@ -2,12 +2,11 @@ package io.shiftleft.js2cpg.core import io.shiftleft.js2cpg.io.FileDefaults import io.shiftleft.js2cpg.io.FileDefaults.VSIX_SUFFIX - -import java.io.File -import io.shiftleft.js2cpg.parser.PackageJsonParser import io.shiftleft.js2cpg.preprocessing.TypescriptTranspiler import scopt.OptionParser +import java.io.File + object Js2cpgArgumentsParser { val HELP: String = "help" val VERSION: String = "version" @@ -46,7 +45,7 @@ object Js2cpgArgumentsParser { class Js2cpgArgumentsParser { - import Js2cpgArgumentsParser._ + import Js2cpgArgumentsParser.* private lazy val banner: String = """ diff --git a/src/main/scala/io/shiftleft/js2cpg/core/Report.scala b/src/main/scala/io/shiftleft/js2cpg/core/Report.scala deleted file mode 100644 index ab967ec20..000000000 --- a/src/main/scala/io/shiftleft/js2cpg/core/Report.scala +++ /dev/null @@ -1,105 +0,0 @@ -package io.shiftleft.js2cpg.core - -import io.shiftleft.js2cpg.utils.TimeUtils -import org.slf4j.LoggerFactory - -import scala.collection.concurrent.TrieMap -import scala.concurrent.duration.Duration - -object Report { - - private val logger = LoggerFactory.getLogger(Report.getClass) - - private type FileName = String - - private type Reports = TrieMap[FileName, ReportEntry] - - private case class ReportEntry( - loc: Long, - parsed: Boolean, - cpgGen: Boolean, - duration: Long, - isConfig: Boolean = false - ) { - def toSeq: Seq[String] = { - val lines = loc.toString - val dur = if (duration == 0) "-" else TimeUtils.pretty(Duration.fromNanos(duration)) - val es6 = if (parsed) "yes" else "no" - val cpg = if (cpgGen) "yes" else "no" - Seq(lines, es6, cpg, dur) - } - } - -} - -class Report { - - import Report._ - - private val reports: Reports = TrieMap.empty - - def print(): Unit = { - - def formatTable(table: Seq[Seq[String]]): String = { - if (table.isEmpty) "" - else { - // Get column widths based on the maximum cell width in each column (+2 for a one character padding on each side) - val colWidths = - table.transpose.map(_.map(cell => if (cell == null) 0 else cell.length).max + 2) - // Format each row - val rows = table.map( - _.zip(colWidths) - .map { case (item, size) => (" %-" + (size - 1) + "s").format(item) } - .mkString("|", "|", "|") - ) - // Formatted separator row, used to separate the header and draw table borders - val separator = colWidths.map("-" * _).mkString("+", "+", "+") - // Put the table together and return - val header = rows.head - val content = rows.tail.take(rows.tail.size - 1) - val footer = rows.tail.last - (separator +: header +: separator +: content :+ separator :+ footer :+ separator) - .mkString("\n") - } - } - - val rows = reports.toSeq - .sortBy(_._1) - .zipWithIndex - .view - .map { - case ((file, sum), index) if sum.isConfig => - s"${index + 1}" +: s"$file (config file)" +: sum.toSeq - case ((file, sum), index) => s"${index + 1}" +: file +: sum.toSeq - } - .toSeq - val numOfReports = reports.size - val header = Seq(Seq("#", "File", "LOC", "Parsed", "Got a CPG", "CPG Gen. Duration")) - val footer = Seq( - Seq( - "Total", - "", - s"${reports.map(_._2.loc).sum}", - s"${reports.count(_._2.parsed)}/$numOfReports", - s"${reports.count(_._2.cpgGen)}/$numOfReports", - "" - ) - ) - val table = header ++ rows ++ footer - logger.info(s"Report:${System.lineSeparator()}" + formatTable(table)) - } - - def addReportInfo( - fileName: FileName, - loc: Long, - parsed: Boolean = false, - cpgGen: Boolean = false, - duration: Long = 0, - isConfig: Boolean = false - ): Unit = - reports(fileName) = ReportEntry(loc, parsed, cpgGen, duration, isConfig) - - def updateReportDuration(fileName: FileName, duration: Long): Unit = - reports.updateWith(fileName)(_.map(_.copy(cpgGen = true, duration = duration))) - -} diff --git a/src/main/scala/io/shiftleft/js2cpg/datastructures/Stack.scala b/src/main/scala/io/shiftleft/js2cpg/datastructures/Stack.scala deleted file mode 100644 index 4d8eca54e..000000000 --- a/src/main/scala/io/shiftleft/js2cpg/datastructures/Stack.scala +++ /dev/null @@ -1,19 +0,0 @@ -package io.shiftleft.js2cpg.datastructures - -import scala.collection.mutable - -object Stack { - - type Stack[StackElement] = mutable.ListBuffer[StackElement] - - implicit class StackWrapper[StackElement](val parentStack: Stack[StackElement]) extends AnyVal { - def push(parent: StackElement): Unit = { - parentStack.prepend(parent) - } - - def pop(): Unit = { - parentStack.remove(0) - } - } - -} diff --git a/src/main/scala/io/shiftleft/js2cpg/datastructures/scope/PendingReference.scala b/src/main/scala/io/shiftleft/js2cpg/datastructures/scope/PendingReference.scala deleted file mode 100644 index a9ed72ccd..000000000 --- a/src/main/scala/io/shiftleft/js2cpg/datastructures/scope/PendingReference.scala +++ /dev/null @@ -1,21 +0,0 @@ -package io.shiftleft.js2cpg.datastructures.scope - -import io.shiftleft.codepropertygraph.generated.nodes.NewNode - -case class PendingReference(variableName: String, referenceNodeId: NewNode, stack: Option[ScopeElement]) { - - def tryResolve(): Option[ResolvedReference] = { - var foundVariableOption = Option.empty[NewNode] - val stackIterator = new ScopeElementIterator(stack) - - while (stackIterator.hasNext && foundVariableOption.isEmpty) { - val scopeElement = stackIterator.next() - foundVariableOption = scopeElement.nameToVariableNode.get(variableName) - } - - foundVariableOption.map { variableNodeId => - ResolvedReference(variableNodeId, this) - } - } - -} diff --git a/src/main/scala/io/shiftleft/js2cpg/datastructures/scope/ResolvedReference.scala b/src/main/scala/io/shiftleft/js2cpg/datastructures/scope/ResolvedReference.scala deleted file mode 100644 index ba7ca7852..000000000 --- a/src/main/scala/io/shiftleft/js2cpg/datastructures/scope/ResolvedReference.scala +++ /dev/null @@ -1,5 +0,0 @@ -package io.shiftleft.js2cpg.datastructures.scope - -import io.shiftleft.codepropertygraph.generated.nodes.NewNode - -case class ResolvedReference(variableNodeId: NewNode, origin: PendingReference) diff --git a/src/main/scala/io/shiftleft/js2cpg/datastructures/scope/Scope.scala b/src/main/scala/io/shiftleft/js2cpg/datastructures/scope/Scope.scala deleted file mode 100644 index 8881a25bc..000000000 --- a/src/main/scala/io/shiftleft/js2cpg/datastructures/scope/Scope.scala +++ /dev/null @@ -1,106 +0,0 @@ -package io.shiftleft.js2cpg.datastructures.scope - -import io.shiftleft.codepropertygraph.generated.nodes.NewNode - -import scala.collection.mutable - -/** Handles the scope stack for tracking identifier to variable relation. - */ -class Scope { - - private val pendingReferences: mutable.Buffer[PendingReference] = - mutable.ListBuffer.empty[PendingReference] - - private var stack = Option.empty[ScopeElement] - - def getScopeHead: Option[ScopeElement] = stack - - def isEmpty: Boolean = stack.isEmpty - - def pushNewMethodScope( - methodFullName: String, - name: String, - scopeNode: NewNode, - capturingRefId: Option[NewNode] - ): Unit = - stack = Some(new MethodScopeElement(methodFullName, capturingRefId, name, scopeNode, surroundingScope = stack)) - - def pushNewBlockScope(scopeNode: NewNode): Unit = { - peek match { - case Some(stackTop) => - stack = Some(new BlockScopeElement(stackTop.subScopeCounter.toString, scopeNode, surroundingScope = stack)) - stackTop.subScopeCounter += 1 - case None => - stack = Some(new BlockScopeElement("0", scopeNode, surroundingScope = stack)) - } - } - - private def peek: Option[ScopeElement] = { - stack - } - - def popScope(): Unit = { - stack = stack.get.surroundingScope - } - - def addVariable(variableName: String, variableNode: NewNode, scopeType: ScopeType): Unit = { - addVariable(stack, variableName, variableNode, scopeType) - } - - def addVariableReference(variableName: String, referenceNode: NewNode): Unit = { - pendingReferences prepend PendingReference(variableName, referenceNode, stack) - } - - private def getEnclosingMethodScope(scopeHead: Option[ScopeElement]): NewNode = { - new ScopeElementIterator(scopeHead) - .collectFirst { case methodScopeElement: MethodScopeElement => - methodScopeElement.scopeNode - } - .getOrElse(throw new RuntimeException("Cannot find method scope.")) - // There are no references outside of methods. Meaning we always find a MethodScope here. - } - - def resolve(unresolvedHandler: (NewNode, String) => (NewNode, ScopeType)): Iterator[ResolvedReference] = { - pendingReferences.iterator.map { pendingReference => - val resolvedReferenceOption = pendingReference.tryResolve() - - resolvedReferenceOption.getOrElse { - val methodScopeNodeId = getEnclosingMethodScope(pendingReference.stack) - val (newVariableNode, scopeType) = - unresolvedHandler(methodScopeNodeId, pendingReference.variableName) - addVariable(pendingReference.stack, pendingReference.variableName, newVariableNode, scopeType) - pendingReference.tryResolve().get - } - } - - } - - private def addVariable( - stack: Option[ScopeElement], - variableName: String, - variableNode: NewNode, - scopeType: ScopeType - ): Unit = { - val scopeToAddTo = scopeType match { - case MethodScope => - new ScopeElementIterator(stack).find(_.isInstanceOf[MethodScopeElement]).get - case _ => stack.get - } - scopeToAddTo.addVariable(variableName, variableNode) - } - -} - -class ScopeElementIterator(start: Option[ScopeElement]) extends Iterator[ScopeElement] { - private var currentScopeElement = start - - override def hasNext: Boolean = { - currentScopeElement.isDefined - } - - override def next(): ScopeElement = { - val result = currentScopeElement.get - currentScopeElement = result.surroundingScope - result - } -} diff --git a/src/main/scala/io/shiftleft/js2cpg/datastructures/scope/ScopeElement.scala b/src/main/scala/io/shiftleft/js2cpg/datastructures/scope/ScopeElement.scala deleted file mode 100644 index aa69c046c..000000000 --- a/src/main/scala/io/shiftleft/js2cpg/datastructures/scope/ScopeElement.scala +++ /dev/null @@ -1,29 +0,0 @@ -package io.shiftleft.js2cpg.datastructures.scope - -import io.shiftleft.codepropertygraph.generated.nodes.NewNode - -import scala.collection.mutable - -/** A single element of a scope stack. - */ -abstract class ScopeElement(val name: String, val scopeNode: NewNode, val surroundingScope: Option[ScopeElement]) { - var subScopeCounter: Int = 0 - val nameToVariableNode: mutable.Map[String, NewNode] = mutable.HashMap.empty - - def addVariable(variableName: String, variableNode: NewNode): Unit = - nameToVariableNode(variableName) = variableNode -} - -class MethodScopeElement( - val methodFullName: String, - val capturingRefId: Option[NewNode], - override val name: String, - override val scopeNode: NewNode, - override val surroundingScope: Option[ScopeElement] -) extends ScopeElement(name, scopeNode, surroundingScope) - -class BlockScopeElement( - override val name: String, - override val scopeNode: NewNode, - override val surroundingScope: Option[ScopeElement] -) extends ScopeElement(name, scopeNode, surroundingScope) diff --git a/src/main/scala/io/shiftleft/js2cpg/datastructures/scope/ScopeType.scala b/src/main/scala/io/shiftleft/js2cpg/datastructures/scope/ScopeType.scala deleted file mode 100644 index f3bb103a0..000000000 --- a/src/main/scala/io/shiftleft/js2cpg/datastructures/scope/ScopeType.scala +++ /dev/null @@ -1,5 +0,0 @@ -package io.shiftleft.js2cpg.datastructures.scope - -sealed trait ScopeType -object MethodScope extends ScopeType -object BlockScope extends ScopeType diff --git a/src/main/scala/io/shiftleft/js2cpg/io/ExternalCommand.scala b/src/main/scala/io/shiftleft/js2cpg/io/ExternalCommand.scala index 726f7656f..39f39dbf8 100644 --- a/src/main/scala/io/shiftleft/js2cpg/io/ExternalCommand.scala +++ b/src/main/scala/io/shiftleft/js2cpg/io/ExternalCommand.scala @@ -3,7 +3,7 @@ package io.shiftleft.js2cpg.io import java.util.concurrent.ConcurrentLinkedQueue import scala.sys.process.{Process, ProcessLogger} import scala.util.{Failure, Success, Try} -import scala.jdk.CollectionConverters._ +import scala.jdk.CollectionConverters.CollectionHasAsScala import org.apache.commons.lang3.StringUtils object ExternalCommand { diff --git a/src/main/scala/io/shiftleft/js2cpg/io/FileDefaults.scala b/src/main/scala/io/shiftleft/js2cpg/io/FileDefaults.scala index d0cadcd86..ef0df5e19 100644 --- a/src/main/scala/io/shiftleft/js2cpg/io/FileDefaults.scala +++ b/src/main/scala/io/shiftleft/js2cpg/io/FileDefaults.scala @@ -89,7 +89,7 @@ object FileDefaults { JSON_LOCK_FILENAME, YARN_LOCK_FILENAME, PNPM_LOCK_FILENAME, - // pnpm workspace config file is not required as we manually descent into sub-project: + // pnpm workspace config file is not required as we manually descent into subprojects: PNPM_WS_FILENAME, NPM_SHRINKWRAP_FILENAME, WEBPACK_CONFIG_FILENAME, diff --git a/src/main/scala/io/shiftleft/js2cpg/io/FileUtils.scala b/src/main/scala/io/shiftleft/js2cpg/io/FileUtils.scala index bc747d8c4..dd7ab1d72 100644 --- a/src/main/scala/io/shiftleft/js2cpg/io/FileUtils.scala +++ b/src/main/scala/io/shiftleft/js2cpg/io/FileUtils.scala @@ -165,7 +165,7 @@ object FileUtils { (positionToLineNumber, positionToFirstPositionInLine) } - final case class FileStatistics(linesOfCode: Long, longestLineLength: Int, containsMarker: Boolean) + final case class FileStatistics(linesOfCode: Int, longestLineLength: Int, containsMarker: Boolean) private def createDecoder(): CharsetDecoder = Codec.UTF8.decoder @@ -173,7 +173,7 @@ object FileUtils { .onUnmappableCharacter(CodingErrorAction.REPLACE) def fileStatistics(lines: Iterator[String]): FileStatistics = { - var linesOfCode = 0L + var linesOfCode = 0 var longestLineLength = 0 var containsMarker = false diff --git a/src/main/scala/io/shiftleft/js2cpg/io/JsFileChecks.scala b/src/main/scala/io/shiftleft/js2cpg/io/JsFileChecks.scala index 1130ee02f..9e2d2eb5f 100644 --- a/src/main/scala/io/shiftleft/js2cpg/io/JsFileChecks.scala +++ b/src/main/scala/io/shiftleft/js2cpg/io/JsFileChecks.scala @@ -1,9 +1,8 @@ package io.shiftleft.js2cpg.io import io.shiftleft.js2cpg.core.Js2cpgArgumentsParser -import io.shiftleft.js2cpg.io.FileDefaults._ +import io.shiftleft.js2cpg.io.FileDefaults.* import io.shiftleft.js2cpg.io.FileUtils.FileStatistics -import io.shiftleft.utils.IOUtils import org.slf4j.LoggerFactory import java.nio.file.Path diff --git a/src/main/scala/io/shiftleft/js2cpg/io/PathFilter.scala b/src/main/scala/io/shiftleft/js2cpg/io/PathFilter.scala index 3c45694d3..577cf63c3 100644 --- a/src/main/scala/io/shiftleft/js2cpg/io/PathFilter.scala +++ b/src/main/scala/io/shiftleft/js2cpg/io/PathFilter.scala @@ -2,7 +2,7 @@ package io.shiftleft.js2cpg.io import better.files.File import io.shiftleft.js2cpg.core.Config -import io.shiftleft.js2cpg.io.FileDefaults._ +import io.shiftleft.js2cpg.io.FileDefaults.* import org.slf4j.LoggerFactory import java.nio.file.{InvalidPathException, Path, Paths} @@ -41,7 +41,7 @@ case class PathFilter( } } - /** We only accept a file if its a regular file and has the appropriate extension. + /** We only accept a file if it is a regular file and has the appropriate extension. * * @param file * the file to inspect diff --git a/src/main/scala/io/shiftleft/js2cpg/parser/DefaultAstVisitor.scala b/src/main/scala/io/shiftleft/js2cpg/parser/DefaultAstVisitor.scala index ce2da7e63..f975e4fdd 100644 --- a/src/main/scala/io/shiftleft/js2cpg/parser/DefaultAstVisitor.scala +++ b/src/main/scala/io/shiftleft/js2cpg/parser/DefaultAstVisitor.scala @@ -1,7 +1,7 @@ package io.shiftleft.js2cpg.parser import com.oracle.js.parser.ir -import com.oracle.js.parser.ir._ +import com.oracle.js.parser.ir.* import com.oracle.js.parser.ir.visitor.NodeVisitor // just a convenience class to make all the methods at hand explicit diff --git a/src/main/scala/io/shiftleft/js2cpg/parser/FreshJsonParser.scala b/src/main/scala/io/shiftleft/js2cpg/parser/FreshJsonParser.scala index dfb8d0f6c..18c000519 100644 --- a/src/main/scala/io/shiftleft/js2cpg/parser/FreshJsonParser.scala +++ b/src/main/scala/io/shiftleft/js2cpg/parser/FreshJsonParser.scala @@ -10,7 +10,7 @@ import io.shiftleft.utils.IOUtils import scala.collection.concurrent.TrieMap import scala.util.Try -import scala.jdk.CollectionConverters._ +import scala.jdk.CollectionConverters.SetHasAsScala object FreshJsonParser { private val logger = LoggerFactory.getLogger(FreshJsonParser.getClass) @@ -50,8 +50,8 @@ object FreshJsonParser { val content = IOUtils.readLinesInFile(freshJsonPath).mkString val json = new ObjectMapper().readTree(content) val dependencyIt = Option(json.get("imports")) - .map(_.fields().asScala) - .getOrElse(Iterator.empty) + .map(_.properties().asScala) + .getOrElse(Set.empty) dependencyIt.collect { case entry if !entry.getKey.startsWith("@") => cleanKey(entry.getKey) -> extractVersion(entry.getValue.asText()) diff --git a/src/main/scala/io/shiftleft/js2cpg/parser/GeneralizingAstVisitor.scala b/src/main/scala/io/shiftleft/js2cpg/parser/GeneralizingAstVisitor.scala index 454258454..e4b0d0cea 100644 --- a/src/main/scala/io/shiftleft/js2cpg/parser/GeneralizingAstVisitor.scala +++ b/src/main/scala/io/shiftleft/js2cpg/parser/GeneralizingAstVisitor.scala @@ -1,6 +1,6 @@ package io.shiftleft.js2cpg.parser -import com.oracle.js.parser.ir._ +import com.oracle.js.parser.ir.* import com.oracle.js.parser.ir.visitor.TranslatorNodeVisitor /** This visitor mapped the enterXXX methods to the visit() to provide an interface which is closer to the original diff --git a/src/main/scala/io/shiftleft/js2cpg/parser/JavaScriptParser.scala b/src/main/scala/io/shiftleft/js2cpg/parser/JavaScriptParser.scala index e6e1a65ba..76085418a 100644 --- a/src/main/scala/io/shiftleft/js2cpg/parser/JavaScriptParser.scala +++ b/src/main/scala/io/shiftleft/js2cpg/parser/JavaScriptParser.scala @@ -2,7 +2,7 @@ package io.shiftleft.js2cpg.parser import com.oracle.js.parser.{ErrorManager, Parser, ParserException, ScriptEnvironment, Source} import com.oracle.js.parser.ir.{ErrorNode, FunctionNode} -import io.shiftleft.js2cpg.utils.SourceWrapper._ +import io.shiftleft.js2cpg.utils.SourceWrapper.* import org.slf4j.LoggerFactory import scala.collection.mutable @@ -12,8 +12,8 @@ object JavaScriptParser { private val logger = LoggerFactory.getLogger(JavaScriptParser.getClass) - // A value >= 13 is internally used by graaljs for the ECMA script version 2022. - // Sadly the do not provide a define for this. + // A value >= 13 is internally used by GraalJS for the ECMA script version 2022. + // Sadly they do not provide a pre-defined constant for this. private val ecmaVersion = 13 private val moduleName = ":program" @@ -74,7 +74,7 @@ object JavaScriptParser { } private def nodeJsFix(jsSource: JsSource): JsSource = { - val lines = jsSource.source.getContent.toString.linesIterator.toSeq + val lines = jsSource.source.getContent.linesIterator.toSeq val replaceIndex = lines.lastIndexWhere(l => importRegex.matches(l.trim())) + 1 val (head, rest) = lines.splitAt(replaceIndex) val fixedCode = (head ++ nodeJsFixWrapper(rest)).mkString("\n") @@ -86,7 +86,7 @@ object JavaScriptParser { val moduleErrorManager = new Js2CpgErrMgr("strict mode") buildParser(jsSource, moduleErrorManager).parseModule(moduleName) match { case null if moduleErrorManager.containsNodeJSInvalidReturn => - // We might have to fix NodeJS code here: + // We might have to fix Node.js code here: val newSource = nodeJsFix(jsSource) val fixed = buildParser(newSource, moduleErrorManager).parseModule(moduleName) if (fixed == null) { @@ -96,7 +96,7 @@ object JavaScriptParser { logger.debug(s"Applied NodeJS fix for file '${jsSource.filePath}'.") (fixed, newSource) } - // if module parsing fails completely we have to fall-back to non-strict mode loosing imports/exports: + // if module parsing fails completely we have to fall back to non-strict mode loosing imports/exports: case null => val nonStrictErrorManager = new Js2CpgErrMgr("non-strict mode") buildParser(jsSource, nonStrictErrorManager).parse() match { diff --git a/src/main/scala/io/shiftleft/js2cpg/parser/JsSource.scala b/src/main/scala/io/shiftleft/js2cpg/parser/JsSource.scala index 7c6afada9..4d3d05545 100644 --- a/src/main/scala/io/shiftleft/js2cpg/parser/JsSource.scala +++ b/src/main/scala/io/shiftleft/js2cpg/parser/JsSource.scala @@ -5,14 +5,14 @@ import better.files.File import com.atlassian.sourcemap.{ReadableSourceMap, ReadableSourceMapImpl} import com.oracle.js.parser.Source import com.oracle.js.parser.ir.Node -import io.shiftleft.js2cpg.io.FileDefaults._ +import io.shiftleft.js2cpg.io.FileDefaults.* import io.shiftleft.js2cpg.io.FileUtils import io.shiftleft.js2cpg.preprocessing.NuxtTranspiler import io.shiftleft.utils.IOUtils import org.apache.commons.lang3.StringUtils import org.slf4j.LoggerFactory -import scala.jdk.CollectionConverters._ +import scala.jdk.CollectionConverters.{ListHasAsScala, SetHasAsScala} import scala.util.{Failure, Success, Try} object JsSource { @@ -30,7 +30,7 @@ object JsSource { class JsSource(val srcDir: File, val projectDir: Path, val source: Source) { - import JsSource._ + import JsSource.* private val absoluteFilePath = (File(projectDir.toAbsolutePath) / originalFilePath).pathAsString private val mapFilePath = absoluteFilePath + ".map" @@ -86,7 +86,7 @@ class JsSource(val srcDir: File, val projectDir: Path, val source: Source) { if (sourceFileName.isEmpty) { srcDir / source.getName } else if (absoluteFilePath.contains(NuxtTranspiler.NUXT_FOLDER) && srcDir.path.compareTo(projectDir) == 0) { - // For nuxt-js transpilation we have the same src and project dir and we need some special handling here + // For nuxt-js transpilation we have the same src and project dir, and we need some special handling here if (sourceFileName.startsWith(WEBPACK_PREFIX)) { val replacedName = FileUtils.cleanPath(sourceFileName.replace(WEBPACK_PREFIX, "")) srcDir / replacedName.substring(replacedName.indexOf("/") + 1) diff --git a/src/main/scala/io/shiftleft/js2cpg/parser/PackageJsonParser.scala b/src/main/scala/io/shiftleft/js2cpg/parser/PackageJsonParser.scala index 040e65d3a..ebdbb562b 100644 --- a/src/main/scala/io/shiftleft/js2cpg/parser/PackageJsonParser.scala +++ b/src/main/scala/io/shiftleft/js2cpg/parser/PackageJsonParser.scala @@ -1,19 +1,16 @@ package io.shiftleft.js2cpg.parser import com.fasterxml.jackson.core.JsonParser - -import java.nio.file.{Path, Paths} -import org.slf4j.LoggerFactory import com.fasterxml.jackson.databind.ObjectMapper import io.shiftleft.js2cpg.io.FileDefaults import io.shiftleft.utils.IOUtils import org.apache.commons.lang3.StringUtils +import org.slf4j.LoggerFactory +import java.nio.file.{Path, Paths} import scala.collection.concurrent.TrieMap -import scala.util.Try -import scala.jdk.CollectionConverters._ -import scala.util.Failure -import scala.util.Success +import scala.jdk.CollectionConverters.SetHasAsScala +import scala.util.{Failure, Success, Try} object PackageJsonParser { private val logger = LoggerFactory.getLogger(PackageJsonParser.getClass) @@ -59,7 +56,7 @@ object PackageJsonParser { val lockDeps = Try { val content = IOUtils.readLinesInFile(lockDepsPath).mkString val packageJson = new ObjectMapper().readTree(content) - val dependencyIt = Option(packageJson.get("dependencies")).map(_.fields().asScala).getOrElse(Iterator.empty) + val dependencyIt = Option(packageJson.get("dependencies")).map(_.properties().asScala).getOrElse(Set.empty) dependencyIt.flatMap { entry => val depName = entry.getKey val versionNode = entry.getValue.get("version") @@ -72,7 +69,7 @@ object PackageJsonParser { val content = IOUtils.readLinesInFile(depsPath).mkString val packageJson = new ObjectMapper().readTree(content) PROJECT_DEPENDENCIES.flatMap { dependency => - val dependencyIt = Option(packageJson.get(dependency)).map(_.fields().asScala).getOrElse(Iterator.empty) + val dependencyIt = Option(packageJson.get(dependency)).map(_.properties().asScala).getOrElse(Set.empty) dependencyIt.map { entry => entry.getKey -> entry.getValue.asText() } }.toMap }.toOption diff --git a/src/main/scala/io/shiftleft/js2cpg/parser/TsConfigJsonParser.scala b/src/main/scala/io/shiftleft/js2cpg/parser/TsConfigJsonParser.scala index befb9f4f9..817d1b34c 100644 --- a/src/main/scala/io/shiftleft/js2cpg/parser/TsConfigJsonParser.scala +++ b/src/main/scala/io/shiftleft/js2cpg/parser/TsConfigJsonParser.scala @@ -6,7 +6,7 @@ import org.slf4j.LoggerFactory import java.nio.file.Path import scala.util.{Failure, Success} -import scala.jdk.CollectionConverters._ +import scala.jdk.CollectionConverters.IteratorHasAsScala object TsConfigJsonParser { diff --git a/src/main/scala/io/shiftleft/js2cpg/passes/AstCreationPass.scala b/src/main/scala/io/shiftleft/js2cpg/passes/AstCreationPass.scala index 145d7b867..ebc31c29d 100644 --- a/src/main/scala/io/shiftleft/js2cpg/passes/AstCreationPass.scala +++ b/src/main/scala/io/shiftleft/js2cpg/passes/AstCreationPass.scala @@ -4,15 +4,14 @@ import java.nio.file.Path import better.files.File import com.oracle.js.parser.Source import com.oracle.js.parser.ir.FunctionNode +import io.joern.x2cpg.utils.{Report, TimeUtils} import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.js2cpg.core.Report import io.shiftleft.js2cpg.astcreation.AstCreator import io.shiftleft.js2cpg.io.{FileUtils, JsFileChecks} import io.shiftleft.js2cpg.parser.{JavaScriptParser, JsSource} import io.shiftleft.passes.ForkJoinParallelCpgPass import org.slf4j.LoggerFactory -import io.shiftleft.js2cpg.utils.SourceWrapper._ -import io.shiftleft.js2cpg.utils.TimeUtils +import io.shiftleft.js2cpg.utils.SourceWrapper.* import scala.util.{Failure, Success, Try} @@ -28,30 +27,25 @@ class AstCreationPass(srcDir: File, filenames: List[(Path, Path)], cpg: Cpg, rep override def generateParts(): Array[(Path, Path)] = filenames.toArray - override def runOnPart(diffGraph: DiffGraphBuilder, filename: (Path, Path)): Unit = { - val (file, fileRoot) = filename - - val parseResult = parse(file, fileRoot) match { + override def runOnPart(diffGraph: DiffGraphBuilder, input: (Path, Path)): Unit = { + val (file, fileRoot) = input + parse(file, fileRoot) match { + case Success(parseResult) => + val usedIdentNodes = collectUsedIdentNodes(parseResult) + val path = parseResult.jsSource.originalFilePath + val (result, duration) = TimeUtils.time { + generateCpg(parseResult, Cpg.newDiffGraphBuilder, usedIdentNodes) + } + result match { + case Success(localDiff) => + logger.info(s"Processed file '$path'") + report.updateReport(path, true, duration) + diffGraph.absorb(localDiff) + case Failure(exception) => + logger.warn(s"Failed to generate CPG for '$path'!", exception) + } case Failure(parseException) => logger.warn(parseException.getMessage) - None - case Success(parseResult) => - Some((parseResult, preAnalyze(parseResult))) - } - - parseResult.map { case (parseResult, usedIdentNodes) => - val (result, duration) = { - TimeUtils.time(generateCpg(parseResult, Cpg.newDiffGraphBuilder, usedIdentNodes)) - } - val path = parseResult.jsSource.originalFilePath - result match { - case Failure(exception) => - logger.warn(s"Failed to generate CPG for '$path'!", exception) - case Success(localDiff) => - logger.info(s"Processed file '$path'") - report.updateReportDuration(path, duration) - diffGraph.absorb(localDiff) - } } } @@ -70,7 +64,7 @@ class AstCreationPass(srcDir: File, filenames: List[(Path, Path)], cpg: Cpg, rep } } - private def preAnalyze(parseResult: ParseResult): Set[String] = { + private def collectUsedIdentNodes(parseResult: ParseResult): Set[String] = { val ast = parseResult.ast val usedIdentNodesPass = new UsedIdentNodesPass() ast.accept(usedIdentNodesPass) diff --git a/src/main/scala/io/shiftleft/js2cpg/passes/BuiltinTypesPass.scala b/src/main/scala/io/shiftleft/js2cpg/passes/BuiltinTypesPass.scala index b68ec4751..f4511a70f 100644 --- a/src/main/scala/io/shiftleft/js2cpg/passes/BuiltinTypesPass.scala +++ b/src/main/scala/io/shiftleft/js2cpg/passes/BuiltinTypesPass.scala @@ -21,7 +21,7 @@ class BuiltinTypesPass(cpg: Cpg) extends CpgPass(cpg) { diffGraph.addNode(namespaceBlock) val orderTracker = new OrderTracker() - Defines.JsTypes.foreach { case typeName: String => + Defines.JsTypes.foreach { typeName => val tpe = NewType() .name(typeName) .fullName(typeName) diff --git a/src/main/scala/io/shiftleft/js2cpg/passes/ConfigPass.scala b/src/main/scala/io/shiftleft/js2cpg/passes/ConfigPass.scala index 3d8410ada..88c1f858d 100644 --- a/src/main/scala/io/shiftleft/js2cpg/passes/ConfigPass.scala +++ b/src/main/scala/io/shiftleft/js2cpg/passes/ConfigPass.scala @@ -1,11 +1,10 @@ package io.shiftleft.js2cpg.passes +import io.joern.x2cpg.utils.{Report, TimeUtils} import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.codepropertygraph.generated.nodes.NewConfigFile -import io.shiftleft.js2cpg.core.Report import io.shiftleft.js2cpg.io.FileDefaults import io.shiftleft.js2cpg.io.FileUtils -import io.shiftleft.js2cpg.utils.TimeUtils import io.shiftleft.passes.ForkJoinParallelCpgPass import io.shiftleft.utils.IOUtils import org.slf4j.{Logger, LoggerFactory} @@ -40,12 +39,12 @@ class ConfigPass(filenames: List[(Path, Path)], cpg: Cpg, report: Report) val localDiff = Cpg.newDiffGraphBuilder logger.debug(s"Adding file '$relativeFile' as config file.") val configNode = NewConfigFile().name(fileName).content(content.mkString("\n")) - report.addReportInfo(fileName, fileStatistics.linesOfCode, parsed = true, cpgGen = true, isConfig = true) + report.addReportInfo(fileName, fileStatistics.linesOfCode, parsed = true) localDiff.addNode(configNode) localDiff } diffGraph.absorb(result) - report.updateReportDuration(fileName, time) + report.updateReport(fileName, true, time) } } diff --git a/src/main/scala/io/shiftleft/js2cpg/passes/PassHelpers.scala b/src/main/scala/io/shiftleft/js2cpg/passes/PassHelpers.scala index 8152fd9b1..d66dd84fd 100644 --- a/src/main/scala/io/shiftleft/js2cpg/passes/PassHelpers.scala +++ b/src/main/scala/io/shiftleft/js2cpg/passes/PassHelpers.scala @@ -1,11 +1,11 @@ package io.shiftleft.js2cpg.passes import com.oracle.js.parser.ir.LiteralNode.ArrayLiteralNode -import com.oracle.js.parser.ir._ +import com.oracle.js.parser.ir.* import scala.annotation.tailrec import scala.collection.mutable -import scala.jdk.CollectionConverters._ +import scala.jdk.CollectionConverters.ListHasAsScala object PassHelpers { diff --git a/src/main/scala/io/shiftleft/js2cpg/passes/PrivateKeyFilePass.scala b/src/main/scala/io/shiftleft/js2cpg/passes/PrivateKeyFilePass.scala index 96fe83288..00372bb21 100644 --- a/src/main/scala/io/shiftleft/js2cpg/passes/PrivateKeyFilePass.scala +++ b/src/main/scala/io/shiftleft/js2cpg/passes/PrivateKeyFilePass.scala @@ -1,7 +1,7 @@ package io.shiftleft.js2cpg.passes +import io.joern.x2cpg.utils.Report import io.shiftleft.codepropertygraph.Cpg -import io.shiftleft.js2cpg.core.Report import io.shiftleft.utils.IOUtils import java.nio.file.Path diff --git a/src/main/scala/io/shiftleft/js2cpg/preprocessing/NuxtTranspiler.scala b/src/main/scala/io/shiftleft/js2cpg/preprocessing/NuxtTranspiler.scala index a0755a1eb..fb16606c2 100644 --- a/src/main/scala/io/shiftleft/js2cpg/preprocessing/NuxtTranspiler.scala +++ b/src/main/scala/io/shiftleft/js2cpg/preprocessing/NuxtTranspiler.scala @@ -39,7 +39,7 @@ object NuxtTranspiler { class NuxtTranspiler(override val config: Config, override val projectPath: Path) extends Transpiler { - import NuxtTranspiler._ + import NuxtTranspiler.* private val logger = LoggerFactory.getLogger(getClass) private val nuxt = Paths.get(projectPath.toString, "node_modules", ".bin", "nuxt").toString diff --git a/src/main/scala/io/shiftleft/js2cpg/preprocessing/TranspilationRunner.scala b/src/main/scala/io/shiftleft/js2cpg/preprocessing/TranspilationRunner.scala index e4f655630..fbd330ea6 100644 --- a/src/main/scala/io/shiftleft/js2cpg/preprocessing/TranspilationRunner.scala +++ b/src/main/scala/io/shiftleft/js2cpg/preprocessing/TranspilationRunner.scala @@ -7,7 +7,7 @@ import com.fasterxml.jackson.databind.node.ArrayNode import com.fasterxml.jackson.databind.node.ObjectNode import io.shiftleft.js2cpg.core.Config import io.shiftleft.js2cpg.io.FileDefaults -import io.shiftleft.js2cpg.io.FileDefaults._ +import io.shiftleft.js2cpg.io.FileDefaults.* import io.shiftleft.js2cpg.io.FileUtils import io.shiftleft.js2cpg.parser.PackageJsonParser import io.shiftleft.utils.IOUtils @@ -28,7 +28,7 @@ class TranspilationRunner(projectPath: Path, tmpTranspileDir: Path, config: Conf private def createTranspilers(): Seq[Transpiler] = { // We always run the following transpilers by default when not stated otherwise in the Config. - // This includes running them for sub-projects. + // This includes running them for subprojects. val baseTranspilers = TranspilerGroup( config, projectPath, @@ -38,7 +38,7 @@ class TranspilationRunner(projectPath: Path, tmpTranspileDir: Path, config: Conf ) ) - // When we got no sub-project, we also run the following ones: + // When we got no subproject, we also run the following ones: if (subDir.isEmpty) { val otherTranspilers = Seq( new VueTranspiler(config, projectPath), diff --git a/src/main/scala/io/shiftleft/js2cpg/preprocessing/TranspilingEnvironment.scala b/src/main/scala/io/shiftleft/js2cpg/preprocessing/TranspilingEnvironment.scala index 92f670bd9..5d22637ed 100644 --- a/src/main/scala/io/shiftleft/js2cpg/preprocessing/TranspilingEnvironment.scala +++ b/src/main/scala/io/shiftleft/js2cpg/preprocessing/TranspilingEnvironment.scala @@ -39,7 +39,7 @@ object TranspilingEnvironment { trait TranspilingEnvironment { self: Transpiler => - import TranspilingEnvironment._ + import TranspilingEnvironment.* private val logger = LoggerFactory.getLogger(getClass) diff --git a/src/main/scala/io/shiftleft/js2cpg/preprocessing/VueTranspiler.scala b/src/main/scala/io/shiftleft/js2cpg/preprocessing/VueTranspiler.scala index dad399398..6996df61a 100644 --- a/src/main/scala/io/shiftleft/js2cpg/preprocessing/VueTranspiler.scala +++ b/src/main/scala/io/shiftleft/js2cpg/preprocessing/VueTranspiler.scala @@ -4,7 +4,7 @@ import better.files.File import io.shiftleft.js2cpg.core.Config import io.shiftleft.js2cpg.io.ExternalCommand import io.shiftleft.js2cpg.io.FileDefaults -import io.shiftleft.js2cpg.io.FileDefaults._ +import io.shiftleft.js2cpg.io.FileDefaults.* import io.shiftleft.js2cpg.io.FileUtils import io.shiftleft.js2cpg.parser.PackageJsonParser import org.slf4j.LoggerFactory @@ -57,7 +57,7 @@ object VueTranspiler { class VueTranspiler(override val config: Config, override val projectPath: Path) extends Transpiler { - import VueTranspiler._ + import VueTranspiler.* private val logger = LoggerFactory.getLogger(getClass) diff --git a/src/main/scala/io/shiftleft/js2cpg/utils/TimeUtils.scala b/src/main/scala/io/shiftleft/js2cpg/utils/TimeUtils.scala deleted file mode 100644 index 647acf6c6..000000000 --- a/src/main/scala/io/shiftleft/js2cpg/utils/TimeUtils.scala +++ /dev/null @@ -1,55 +0,0 @@ -package io.shiftleft.js2cpg.utils - -import java.util.Locale -import scala.concurrent.duration._ - -object TimeUtils { - - /** Measures elapsed time for executing a block in nanoseconds */ - def time[R](block: => R): (R, Long) = { - val t0 = System.nanoTime() - val result = block - val t1 = System.nanoTime() - val elapsed = t1 - t0 - (result, elapsed) - } - - /** Selects most appropriate TimeUnit for given duration and formats it accordingly */ - def pretty(duration: Duration): String = { - duration match { - case d: FiniteDuration => - val nanos = d.toNanos - val unit = chooseUnit(nanos) - val value = nanos.toDouble / NANOSECONDS.convert(1, unit) - - s"%.4g %s".formatLocal(Locale.ROOT, value, abbreviate(unit)) - - case Duration.MinusInf => s"-∞ (minus infinity)" - case Duration.Inf => s"∞ (infinity)" - case _ => "undefined" - } - } - - private def chooseUnit(nanos: Long): TimeUnit = { - val d = nanos.nanos - - if (d.toDays > 0) DAYS - else if (d.toHours > 0) HOURS - else if (d.toMinutes > 0) MINUTES - else if (d.toSeconds > 0) SECONDS - else if (d.toMillis > 0) MILLISECONDS - else if (d.toMicros > 0) MICROSECONDS - else NANOSECONDS - } - - private def abbreviate(unit: TimeUnit): String = unit match { - case NANOSECONDS => "ns" - case MICROSECONDS => "μs" - case MILLISECONDS => "ms" - case SECONDS => "s" - case MINUTES => "min" - case HOURS => "h" - case DAYS => "d" - } - -} diff --git a/src/test/scala/io/shiftleft/js2cpg/io/EmScriptenCleanerTest.scala b/src/test/scala/io/shiftleft/js2cpg/io/EmScriptenCleanerTest.scala index 5a763d30a..923156137 100644 --- a/src/test/scala/io/shiftleft/js2cpg/io/EmScriptenCleanerTest.scala +++ b/src/test/scala/io/shiftleft/js2cpg/io/EmScriptenCleanerTest.scala @@ -11,8 +11,7 @@ class EmScriptenCleanerTest extends AnyWordSpec with Matchers { """ |console.log("Hello"); |console.log("World!");""".stripMargin - val result = - EmScriptenCleaner.clean(code.linesIterator.toSeq).mkString("\n") + val result = EmScriptenCleaner.clean(code.linesIterator.toSeq).mkString("\n") result shouldBe code } @@ -22,8 +21,7 @@ class EmScriptenCleanerTest extends AnyWordSpec with Matchers { |// EMSCRIPTEN_START_FUNCS |console.log("Hello"); |console.log("World!");""".stripMargin - val result = - EmScriptenCleaner.clean(code.linesIterator.toSeq).mkString("\n") + val result = EmScriptenCleaner.clean(code.linesIterator.toSeq).mkString("\n") result shouldBe code } @@ -33,8 +31,7 @@ class EmScriptenCleanerTest extends AnyWordSpec with Matchers { |console.log("Hello"); |console.log("World!"); |// EMSCRIPTEN_END_FUNCS""".stripMargin - val result = - EmScriptenCleaner.clean(code.linesIterator.toSeq).mkString("\n") + val result = EmScriptenCleaner.clean(code.linesIterator.toSeq).mkString("\n") result shouldBe code } @@ -45,8 +42,7 @@ class EmScriptenCleanerTest extends AnyWordSpec with Matchers { |console.log("Hello"); |console.log("World!"); |// EMSCRIPTEN_START_FUNCS""".stripMargin - val result = - EmScriptenCleaner.clean(code.linesIterator.toSeq).mkString("\n") + val result = EmScriptenCleaner.clean(code.linesIterator.toSeq).mkString("\n") result shouldBe code } @@ -70,8 +66,7 @@ class EmScriptenCleanerTest extends AnyWordSpec with Matchers { | |otherCode(); |foobar();""".stripMargin - val result = - EmScriptenCleaner.clean(code.linesIterator.toSeq).mkString("\n") + val result = EmScriptenCleaner.clean(code.linesIterator.toSeq).mkString("\n") result shouldBe expected } } diff --git a/src/test/scala/io/shiftleft/js2cpg/io/ExcludeTest.scala b/src/test/scala/io/shiftleft/js2cpg/io/ExcludeTest.scala index 451c6e8d5..2cc1ae6cd 100644 --- a/src/test/scala/io/shiftleft/js2cpg/io/ExcludeTest.scala +++ b/src/test/scala/io/shiftleft/js2cpg/io/ExcludeTest.scala @@ -3,7 +3,7 @@ package io.shiftleft.js2cpg.io import better.files.File import io.shiftleft.codepropertygraph.Cpg import io.shiftleft.js2cpg.core.{Js2cpgArgumentsParser, Js2CpgMain} -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import org.scalatest.matchers.should.Matchers import org.scalatest.prop.TableDrivenPropertyChecks @@ -19,8 +19,8 @@ object ExcludeTest { class ExcludeTest extends AnyWordSpec with Matchers with TableDrivenPropertyChecks { - import ExcludeTest._ - import Js2cpgArgumentsParser._ + import ExcludeTest.* + import Js2cpgArgumentsParser.* private val projectUnderTestPath = File(getClass.getResource("/excludes").toURI).pathAsString diff --git a/src/test/scala/io/shiftleft/js2cpg/io/PrivateModulesTest.scala b/src/test/scala/io/shiftleft/js2cpg/io/PrivateModulesTest.scala index e5bd827b2..eb80c5a5a 100644 --- a/src/test/scala/io/shiftleft/js2cpg/io/PrivateModulesTest.scala +++ b/src/test/scala/io/shiftleft/js2cpg/io/PrivateModulesTest.scala @@ -3,7 +3,7 @@ package io.shiftleft.js2cpg.io import better.files.File import io.shiftleft.codepropertygraph.Cpg import io.shiftleft.js2cpg.core.Js2CpgMain -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec diff --git a/src/test/scala/io/shiftleft/js2cpg/parser/ParserTest.scala b/src/test/scala/io/shiftleft/js2cpg/parser/ParserTest.scala index 3c9a7728c..cc7d7fb97 100644 --- a/src/test/scala/io/shiftleft/js2cpg/parser/ParserTest.scala +++ b/src/test/scala/io/shiftleft/js2cpg/parser/ParserTest.scala @@ -24,7 +24,7 @@ class ParserTest extends AnyWordSpec with Matchers { nodes.collect { case callNode: CallNode if callNode.getFunction.toString == "require" => callNode }.size shouldBe 1 - // but its no module any more + // but its no module anymore ast.isModule shouldBe false } diff --git a/src/test/scala/io/shiftleft/js2cpg/passes/AbstractPassTest.scala b/src/test/scala/io/shiftleft/js2cpg/passes/AbstractPassTest.scala index c2ba885ed..8376088e7 100644 --- a/src/test/scala/io/shiftleft/js2cpg/passes/AbstractPassTest.scala +++ b/src/test/scala/io/shiftleft/js2cpg/passes/AbstractPassTest.scala @@ -2,16 +2,13 @@ package io.shiftleft.js2cpg.passes import better.files.File import io.joern.x2cpg.X2Cpg.newEmptyCpg -import io.shiftleft.codepropertygraph.generated.EdgeTypes +import io.joern.x2cpg.utils.Report import io.shiftleft.codepropertygraph.generated.nodes.Dependency import io.shiftleft.codepropertygraph.Cpg -import io.shiftleft.js2cpg.core.Report import io.shiftleft.semanticcpg.language.* import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec -import scala.jdk.CollectionConverters.* - abstract class AbstractPassTest extends AnyWordSpec with Matchers { protected abstract class Fixture diff --git a/src/test/scala/io/shiftleft/js2cpg/passes/BuiltinTypesPassTest.scala b/src/test/scala/io/shiftleft/js2cpg/passes/BuiltinTypesPassTest.scala index d0de1884c..209ac3a89 100644 --- a/src/test/scala/io/shiftleft/js2cpg/passes/BuiltinTypesPassTest.scala +++ b/src/test/scala/io/shiftleft/js2cpg/passes/BuiltinTypesPassTest.scala @@ -1,7 +1,7 @@ package io.shiftleft.js2cpg.passes import io.shiftleft.codepropertygraph.generated.Cpg -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class BuiltinTypesPassTest extends AbstractPassTest { @@ -14,7 +14,7 @@ class BuiltinTypesPassTest extends AbstractPassTest { } "create types and type decls correctly" in { - Defines.JsTypes.foreach { case typeName: String => + Defines.JsTypes.foreach { typeName => val typeDeclNodes = cpg.typeDecl(typeName).l typeDeclNodes should have length 1 val typeDeclNode = typeDeclNodes.head diff --git a/src/test/scala/io/shiftleft/js2cpg/passes/CfgCreationPassTest.scala b/src/test/scala/io/shiftleft/js2cpg/passes/CfgCreationPassTest.scala index daeb6e623..362ee8b8d 100644 --- a/src/test/scala/io/shiftleft/js2cpg/passes/CfgCreationPassTest.scala +++ b/src/test/scala/io/shiftleft/js2cpg/passes/CfgCreationPassTest.scala @@ -2,10 +2,10 @@ package io.shiftleft.js2cpg.passes import better.files.File import io.shiftleft.codepropertygraph.generated.* -import io.shiftleft.js2cpg.core.Report import io.shiftleft.semanticcpg.language.* import io.joern.x2cpg.passes.controlflow.CfgCreationPass import io.joern.x2cpg.passes.controlflow.cfgcreation.Cfg.* +import io.joern.x2cpg.utils.Report import io.shiftleft.codepropertygraph.generated.nodes.{AstNodeBase, CfgNode, Method} import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec @@ -1300,7 +1300,7 @@ class CfgCreationPassTest extends AnyWordSpec with Matchers { private def matchCode(node: CfgNode, code: String): Boolean = node match { case m: Method => m.name == code case astNode: AstNodeBase => astNode.code == code - case _ => false + case null => false } // index is zero based and describes which node to take if multiple node match the code string. diff --git a/src/test/scala/io/shiftleft/js2cpg/passes/ClassesAstCreationPassTest.scala b/src/test/scala/io/shiftleft/js2cpg/passes/ClassesAstCreationPassTest.scala index 825b6da0d..7465f3b40 100644 --- a/src/test/scala/io/shiftleft/js2cpg/passes/ClassesAstCreationPassTest.scala +++ b/src/test/scala/io/shiftleft/js2cpg/passes/ClassesAstCreationPassTest.scala @@ -1,11 +1,7 @@ package io.shiftleft.js2cpg.passes -import io.shiftleft.codepropertygraph.Cpg import io.shiftleft.codepropertygraph.generated.* import io.shiftleft.codepropertygraph.generated.nodes.* -import io.shiftleft.js2cpg.core.Config -import io.shiftleft.js2cpg.io.FileDefaults.JS_SUFFIX -import io.shiftleft.js2cpg.io.FileUtils import io.shiftleft.semanticcpg.language.* class ClassesAstCreationPassTest extends AbstractPassTest { diff --git a/src/test/scala/io/shiftleft/js2cpg/passes/ConfigPassTest.scala b/src/test/scala/io/shiftleft/js2cpg/passes/ConfigPassTest.scala index 02e9e7c48..e4177f903 100644 --- a/src/test/scala/io/shiftleft/js2cpg/passes/ConfigPassTest.scala +++ b/src/test/scala/io/shiftleft/js2cpg/passes/ConfigPassTest.scala @@ -1,9 +1,9 @@ package io.shiftleft.js2cpg.passes -import io.shiftleft.js2cpg.core.Report import io.shiftleft.codepropertygraph.generated.Cpg import io.shiftleft.semanticcpg.language.* import better.files.File +import io.joern.x2cpg.utils.Report import io.shiftleft.js2cpg.io.FileDefaults import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec diff --git a/src/test/scala/io/shiftleft/js2cpg/passes/DependenciesPassTest.scala b/src/test/scala/io/shiftleft/js2cpg/passes/DependenciesPassTest.scala index ec08cd93e..346df888f 100644 --- a/src/test/scala/io/shiftleft/js2cpg/passes/DependenciesPassTest.scala +++ b/src/test/scala/io/shiftleft/js2cpg/passes/DependenciesPassTest.scala @@ -2,12 +2,13 @@ package io.shiftleft.js2cpg.passes import better.files.File import io.joern.x2cpg.X2Cpg.newEmptyCpg +import io.joern.x2cpg.utils.Report import io.shiftleft.codepropertygraph.Cpg -import io.shiftleft.js2cpg.core.{Config, Report} +import io.shiftleft.js2cpg.core.Config import io.shiftleft.js2cpg.io.FileDefaults import io.shiftleft.js2cpg.parser.PackageJsonParser import io.shiftleft.js2cpg.preprocessing.TypescriptTranspiler -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* class DependenciesPassTest extends AbstractPassTest { diff --git a/src/test/scala/io/shiftleft/js2cpg/passes/JsMetaDataPassTest.scala b/src/test/scala/io/shiftleft/js2cpg/passes/JsMetaDataPassTest.scala index 9d132882b..fe4d0bdfb 100644 --- a/src/test/scala/io/shiftleft/js2cpg/passes/JsMetaDataPassTest.scala +++ b/src/test/scala/io/shiftleft/js2cpg/passes/JsMetaDataPassTest.scala @@ -3,8 +3,6 @@ package io.shiftleft.js2cpg.passes import io.shiftleft.codepropertygraph.generated.{Cpg, Languages} import io.shiftleft.semanticcpg.language.* -import scala.jdk.CollectionConverters._ - class JsMetaDataPassTest extends AbstractPassTest { "MetaDataPass" should { diff --git a/src/test/scala/io/shiftleft/js2cpg/passes/MixedAstCreationPassTest.scala b/src/test/scala/io/shiftleft/js2cpg/passes/MixedAstCreationPassTest.scala index 395463728..4ecb20879 100644 --- a/src/test/scala/io/shiftleft/js2cpg/passes/MixedAstCreationPassTest.scala +++ b/src/test/scala/io/shiftleft/js2cpg/passes/MixedAstCreationPassTest.scala @@ -1,11 +1,7 @@ package io.shiftleft.js2cpg.passes -import io.shiftleft.codepropertygraph.Cpg import io.shiftleft.codepropertygraph.generated.* import io.shiftleft.codepropertygraph.generated.nodes.* -import io.shiftleft.js2cpg.core.Config -import io.shiftleft.js2cpg.io.FileDefaults.JS_SUFFIX -import io.shiftleft.js2cpg.io.FileUtils import io.shiftleft.semanticcpg.language.* class MixedAstCreationPassTest extends AbstractPassTest { diff --git a/src/test/scala/io/shiftleft/js2cpg/passes/SimpleAstCreationPassTest.scala b/src/test/scala/io/shiftleft/js2cpg/passes/SimpleAstCreationPassTest.scala index beab49894..155e841a7 100644 --- a/src/test/scala/io/shiftleft/js2cpg/passes/SimpleAstCreationPassTest.scala +++ b/src/test/scala/io/shiftleft/js2cpg/passes/SimpleAstCreationPassTest.scala @@ -1,11 +1,7 @@ package io.shiftleft.js2cpg.passes -import io.shiftleft.codepropertygraph.Cpg import io.shiftleft.codepropertygraph.generated.* import io.shiftleft.codepropertygraph.generated.nodes.* -import io.shiftleft.js2cpg.core.Config -import io.shiftleft.js2cpg.io.FileDefaults.JS_SUFFIX -import io.shiftleft.js2cpg.io.FileUtils import io.shiftleft.semanticcpg.language.* class SimpleAstCreationPassTest extends AbstractPassTest { diff --git a/src/test/scala/io/shiftleft/js2cpg/preprocessing/TranspilationRunnerTest.scala b/src/test/scala/io/shiftleft/js2cpg/preprocessing/TranspilationRunnerTest.scala index a89e2a124..9be0bf2a6 100644 --- a/src/test/scala/io/shiftleft/js2cpg/preprocessing/TranspilationRunnerTest.scala +++ b/src/test/scala/io/shiftleft/js2cpg/preprocessing/TranspilationRunnerTest.scala @@ -6,7 +6,7 @@ import io.shiftleft.js2cpg.core.Config import io.shiftleft.js2cpg.core.Js2CpgMain import io.shiftleft.js2cpg.io.FileDefaults.JS_SUFFIX import io.shiftleft.js2cpg.io.FileUtils -import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.* import org.scalatest.matchers.should.Matchers import org.scalatest.tags.Slow import org.scalatest.wordspec.AnyWordSpec