Add async generators support via AST transformation (#1657)

Summary:
This PR introduces support for async generators through AST transformations. In detail:

- Added AsyncGenerator.cpp to transform asynchronous generators into standard generators using AST transformations. This implementation mirrors the logic found in [Babel ](https://babeljs.io/docs/babel-plugin-transform-async-generator-functions).
-  Introduced an EnableAsyncGenerators flag to manage this functionality.

Pull Request resolved: #1657

Test Plan: - Added unit tests

Reviewed By: tmikov

Differential Revision: D78378721

Pulled By: avp

fbshipit-source-id: cb67810823e2a1478b562ad3f12c5a5dfbf3f754

Author: trossimel-sc

API/hermes/hermes.cpp

@@ -241,6 +241,8 @@ class HermesRuntimeImpl final : public HermesRuntime,
 
     compileFlags_.enableGenerator = runtimeConfig.getEnableGenerator();
     compileFlags_.enableES6BlockScoping = runtimeConfig.getES6BlockScoping();
+    compileFlags_.enableAsyncGenerators =
+        runtimeConfig.getEnableAsyncGenerators();
     compileFlags_.emitAsyncBreakCheck =
         runtimeConfig.getAsyncBreakCheckInEval();
     runtime_.addCustomRootsFunction(

include/hermes/AST/AsyncGenerator.h

@@ -0,0 +1,21 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#ifndef HERMES_AST_ASYNCGENERATORS_H
+#define HERMES_AST_ASYNCGENERATORS_H
+
+#include "hermes/AST/ESTree.h"
+
+namespace hermes {
+
+/// Recursively transforms the ESTree Node tree such that async generators
+/// are converted into generators
+ESTree::Node *transformAsyncGenerators(Context &context, ESTree::Node *node);
+
+} // namespace hermes
+
+#endif

include/hermes/AST/Context.h

@@ -242,6 +242,9 @@ class Context {
   /// Whether to parse TypeScript syntax.
   bool parseTS_{false};
 
+  /// Whether to enable support for async generators
+  bool enableAsyncGenerators_{false};
+
   /// Whether to enable support for ES6 block scoping.
   /// TODO: This is intended to provide a temporary way to configure block
   ///       scoping until we have debugger support for it.
@@ -433,6 +436,14 @@ class Context {
     return parseTS_;
   }
 
+  void setEnableAsyncGenerators(bool enableAsyncGenerators) {
+    enableAsyncGenerators_ = enableAsyncGenerators;
+  }
+
+  bool getEnableAsyncGenerators() const {
+    return enableAsyncGenerators_;
+  }
+
   void setEnableES6BlockScoping(bool enableES6BlockScoping) {
     enableES6BlockScoping_ = enableES6BlockScoping;
   }

include/hermes/AST/TransformationsBase.h

@@ -0,0 +1,155 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#pragma once
+
+#include "hermes/AST/RecursiveVisitor.h"
+#include "hermes/Parser/JSLexer.h"
+#include "llvh/ADT/StringRef.h"
+
+namespace {
+using namespace hermes;
+
+/// Mutable vector that helps dealing with arrays of nodes safely.
+/// Once done with the vector, it can create an ESTree::NodeList
+/// representation which is used by the ESTree API in several places.
+class NodeVector {
+ public:
+  using Storage = llvh::SmallVector<ESTree::Node *, 8>;
+
+  NodeVector() = default;
+  NodeVector(std::initializer_list<ESTree::Node *> nodes) {
+    for (auto &node : nodes) {
+      _storage.push_back(node);
+    }
+  }
+
+  NodeVector(ESTree::NodeList &list) {
+    for (auto &node : list) {
+      _storage.push_back(&node);
+    }
+  }
+
+  ~NodeVector() = default;
+
+  size_t size() const {
+    return _storage.size();
+  }
+
+  Storage::const_iterator begin() const {
+    return _storage.begin();
+  }
+
+  Storage::const_iterator end() const {
+    return _storage.end();
+  }
+
+  void append(ESTree::Node *node) {
+    _storage.emplace_back(node);
+  }
+
+  void prepend(ESTree::Node *node) {
+    _storage.insert(_storage.begin(), node);
+  }
+
+  ESTree::NodeList toNodeList() const {
+    ESTree::NodeList nodeList;
+    for (auto &node : _storage) {
+      nodeList.push_back(*node);
+    }
+    return nodeList;
+  }
+
+ private:
+  Storage _storage;
+};
+
+class TransformationsBase
+    : public ESTree::RecursionDepthTracker<TransformationsBase> {
+ public:
+  static constexpr bool kEnableNodeListMutation = true;
+
+  TransformationsBase(Context &context)
+      : context_(context),
+        identVar_(context.getIdentifier("var").getUnderlyingPointer()) {}
+
+  void recursionDepthExceeded(ESTree::Node *n) {
+    context_.getSourceErrorManager().error(
+        n->getEndLoc(), "Too many nested expressions/statements/declarations");
+  }
+
+ protected:
+  Context &context_;
+  UniqueString *const identVar_;
+
+  void doCopyLocation(ESTree::Node *src, ESTree::Node *dest) {
+    if (src != nullptr) {
+      dest->setStartLoc(src->getStartLoc());
+      dest->setEndLoc(src->getEndLoc());
+      dest->setDebugLoc(src->getDebugLoc());
+    }
+  }
+
+  template <typename T>
+  T *copyLocation(ESTree::Node *src, T *dest) {
+    doCopyLocation(src, dest);
+    return dest;
+  }
+
+  template <typename T, typename... Args>
+  T *createTransformedNode(ESTree::Node *src, Args &&...args) {
+    auto *node = new (context_) T(std::forward<Args>(args)...);
+    return copyLocation(src, node);
+  }
+
+  ESTree::IdentifierNode *makeIdentifierNode(
+      ESTree::Node *srcNode,
+      UniqueString *name) {
+    return createTransformedNode<ESTree::IdentifierNode>(
+        srcNode, name, nullptr, false);
+  }
+
+  ESTree::IdentifierNode *makeIdentifierNode(
+      ESTree::Node *srcNode,
+      llvh::StringRef name) {
+    return makeIdentifierNode(
+        srcNode, context_.getIdentifier(name).getUnderlyingPointer());
+  }
+
+  ESTree::Node *makeSingleVarDecl(
+      ESTree::Node *srcNode,
+      ESTree::Node *identifier,
+      ESTree::Node *value) {
+    auto *variableDeclarator =
+        createTransformedNode<ESTree::VariableDeclaratorNode>(
+            srcNode, value, identifier);
+    ESTree::NodeList variableList;
+    variableList.push_back(*variableDeclarator);
+    return createTransformedNode<ESTree::VariableDeclarationNode>(
+        srcNode, identVar_, std::move(variableList));
+  }
+
+  ESTree::Node *makeHermesInternalCall(
+      ESTree::Node *srcNode,
+      llvh::StringRef methodName,
+      const NodeVector &parameters) {
+    auto hermesInternalIdentifier = getHermesInternalIdentifier(srcNode);
+    auto methodIdentifier = makeIdentifierNode(srcNode, methodName);
+
+    auto *getPropertyNode = createTransformedNode<ESTree::MemberExpressionNode>(
+        srcNode, hermesInternalIdentifier, methodIdentifier, false);
+    return createTransformedNode<ESTree::CallExpressionNode>(
+        srcNode, getPropertyNode, nullptr, parameters.toNodeList());
+  }
+
+  virtual ESTree::Node *getHermesInternalIdentifier(ESTree::Node *srcNode) {
+    return nullptr;
+  };
+
+  virtual ~TransformationsBase() = default;
+};
+} // namespace

include/hermes/BCGen/HBC/HBC.h

@@ -68,6 +68,8 @@ struct CompileFlags {
   bool enableGenerator{true};
   /// Enable ES6 block scoping support
   bool enableES6BlockScoping{false};
+  /// Enable async generators support
+  bool enableAsyncGenerators{false};
   /// Define the output format of the generated bytecode. For instance, whether
   /// the bytecode is intended for execution or serialisation.
   OutputFormatKind format{Execute};

include/hermes/Runtime/Libhermes.h

@@ -18,6 +18,7 @@ const char libhermes[] =
 "var Error;"
 "var Function;"
 "var HermesInternal;"
+"var HermesAsyncIteratorsInternal;"
 "var JSON;"
 "var Map;"
 "var Math;"

include/hermes/Utils/CompilerRuntimeFlags.h

@@ -48,6 +48,12 @@ struct CompilerRuntimeFlags {
       llvh::cl::init(false),
       llvh::cl::desc("Enable support for ES6 block scoping"),
       llvh::cl::Hidden};
+
+  llvh::cl::opt<bool> EnableAsyncGenerators{
+      "Xasync-generators",
+      llvh::cl::init(false),
+      llvh::cl::desc("Enable support for async generators"),
+      llvh::cl::Hidden};
 };
 
 } // namespace hermes

include/hermes/VM/Runtime.h

@@ -964,6 +964,10 @@ class Runtime : public RuntimeBase, public HandleRootOwner {
     return hasES6Proxy_;
   }
 
+  bool hasAsyncGenerators() const {
+    return hasAsyncGenerators_;
+  }
+
   bool hasES6BlockScoping() const {
     return hasES6BlockScoping_;
   }
@@ -1206,6 +1210,9 @@ class Runtime : public RuntimeBase, public HandleRootOwner {
   /// Set to true if we should enable ES6 Proxy.
   const bool hasES6Proxy_;
 
+  /// Set to true if we should enable async generators.
+  const bool hasAsyncGenerators_;
+
   /// Set to true if we should enable ES6 block scoping.
   const bool hasES6BlockScoping_;