[clr-interp] Add logic to fix tail-call issues (#122618)

- Detect the case where tail calls are not immediately followed by ret,
and fail the tail-call.
- Detect the attempted usage of tail-calls from within synchronized
methods or from within reverse p/invoke methods
- Use the jit interface canTailCall api when a tail-call is requested
- Enable tail-calls for calli instructions

Update the TailcallVerifyWithPrefix test to skip tests which depend on
the jit performing implicit tail-calls when optimizing when running
under the interpreter

This fixes the TailcallVerifyWithPrefix and more_tailcalls test

Author: davidwrighton

src/coreclr/interpreter/compiler.cpp

@@ -4004,7 +4004,7 @@ void InterpCompiler::EmitPushLdvirtftn(int thisVar, CORINFO_RESOLVED_TOKEN* pRes
     m_pLastNewIns->info.pCallInfo->pCallArgs = callArgs;
 }
 
-static bool DisallowTailCall(CORINFO_SIG_INFO* callerSig, CORINFO_SIG_INFO* calleeSig)
+bool InterpCompiler::DisallowTailCall(CORINFO_SIG_INFO* callerSig, CORINFO_SIG_INFO* calleeSig)
 {
     // We allow only the return value types to differ between caller and callee as long as their stack types are the same.
     // In principle we could allow more differences (e.g. I8 coercion to I4, or O to I) but for now we keep it simple.
@@ -4024,6 +4024,16 @@ static bool DisallowTailCall(CORINFO_SIG_INFO* callerSig, CORINFO_SIG_INFO* call
         // Disallow tail calls from async methods for now
         return true;
     }
+    if (m_isSynchronized)
+    {
+        // Disallow tail calls from synchronized methods
+        return true;
+    }
+    if (m_corJitFlags.IsSet(CORJIT_FLAGS::CORJIT_FLAG_REVERSE_PINVOKE))
+    {
+        // Disallow tail calls from reverse pinvoke methods
+        return true;
+    }
     return false;
 }
 
@@ -4488,7 +4498,14 @@ void InterpCompiler::EmitCall(CORINFO_RESOLVED_TOKEN* pConstrainedToken, bool re
 #endif
     }
 
-    if (tailcall && DisallowTailCall(&m_methodInfo->args, &callInfo.sig))
+    if (tailcall && (
+        DisallowTailCall(&m_methodInfo->args, &callInfo.sig) // Disallow tail-calls for code gen reasons
+        || !m_compHnd->canTailCall(m_methodHnd, // Disallow tail calls due to rules specified by the VM
+                                   isCalli ? (CORINFO_METHOD_HANDLE)NULL : callInfo.hMethod, // The method we are attempting to call logically
+                                   isCalli ? (CORINFO_METHOD_HANDLE)NULL : (callInfo.kind == CORINFO_CALL ? callInfo.hMethod : (CORINFO_METHOD_HANDLE)NULL),
+                                   true) // The method we are calling exactly. We only know this if it's a non-virtual call
+        || (!isJmp && *m_ip != CEE_RET) // Disallow tailcalls that are not immediately before a ret
+        ))
     {
         if (isJmp)
         {
@@ -9043,7 +9060,7 @@ void InterpCompiler::GenerateCode(CORINFO_METHOD_INFO* methodInfo)
                 tailcall = false;
                 break;
             case CEE_CALLI:
-                EmitCall(NULL /*pConstrainedToken*/, false /* readonly*/, false /* tailcall*/, false /*newObj*/, true /*isCalli*/);
+                EmitCall(NULL /*pConstrainedToken*/, false /* readonly*/, tailcall /* tailcall*/, false /*newObj*/, true /*isCalli*/);
                 pConstrainedToken = NULL;
                 readonly = false;
                 tailcall = false;

src/coreclr/interpreter/compiler.h

@@ -872,6 +872,7 @@ class InterpCompiler
     void PushInterpType(InterpType interpType, CORINFO_CLASS_HANDLE clsHnd);
     void PushTypeVT(CORINFO_CLASS_HANDLE clsHnd, int size);
     void ConvertFloatingPointStackEntryToStackType(StackInfo* entry, StackType type);
+    bool DisallowTailCall(CORINFO_SIG_INFO* callerSig, CORINFO_SIG_INFO* calleeSig);
 
     // Opcode peeps
     bool    FindAndApplyPeep(OpcodePeep* Peeps[]);

src/tests/JIT/opt/Tailcall/TailcallVerifyWithPrefix.il

@@ -28,6 +28,10 @@
   .publickeytoken = (B0 3F 5F 7F 11 D5 0A 3A )
   .ver 4:0:0:0
 }
+.assembly extern TestLibrary
+{
+  .ver 0:0:0:0
+}
 .assembly TailcallVerifyWithPrefix
 {
   .custom instance void [mscorlib]System.Reflection.AssemblyDescriptionAttribute::.ctor(string) = ( 01 00 00 00 00 )
@@ -3341,6 +3345,14 @@
     + "one - ReturnType: void; Callee<string>: Arguments: None - ReturnType: v"
     + "oid"
     IL_0005:  call       void [System.Console]System.Console::WriteLine(string)
+    // This test assumes that regular calls will be converted to tail-calls when possible
+    call bool [TestLibrary]TestLibrary.Utilities::get_IsCoreClrInterpreter()
+    ldc.i4.0
+    beq IL_000a
+    ldstr "Skipping test on interpreter, as it doesn't support converting regular calls into tail-calls"
+    call void [System.Console]System.Console::WriteLine(string)
+    ldc.i4.s 100
+    ret
     IL_000a:  ldc.i4.s   100
     IL_000c:  stsfld     int32 TailcallVerify.Condition10::Result
     .try
@@ -3457,6 +3469,14 @@
     + "alueType3Bytes>: Caller: Arguments: None - ReturnType: void; Callee: Ar"
     + "guments: None - ReturnType: void"
     IL_0023:  call       void [System.Console]System.Console::WriteLine(string)
+    // This test assumes that regular calls will be converted to tail-calls when possible
+    call bool [TestLibrary]TestLibrary.Utilities::get_IsCoreClrInterpreter()
+    ldc.i4.0
+    beq IL_0028
+    ldstr "Skipping test on interpreter, as it doesn't support converting regular calls into tail-calls"
+    call void [System.Console]System.Console::WriteLine(string)
+    ldc.i4.s 100
+    ret
     IL_0028:  ldc.i4.s   100
     IL_002a:  stsfld     int32 TailcallVerify.Condition10::Result
     .try
@@ -3536,6 +3556,14 @@
     + "alueType3Bytes>: Caller: Arguments: string, ValueType3Bytes - ReturnTyp"
     + "e: void; Callee: Arguments: string - ReturnType: void"
     IL_0023:  call       void [System.Console]System.Console::WriteLine(string)
+    // This test assumes that regular calls will be converted to tail-calls when possible
+    call bool [TestLibrary]TestLibrary.Utilities::get_IsCoreClrInterpreter()
+    ldc.i4.0
+    beq IL_0028
+    ldstr "Skipping test on interpreter, as it doesn't support converting regular calls into tail-calls"
+    call void [System.Console]System.Console::WriteLine(string)
+    ldc.i4.s 100
+    ret
     IL_0028:  ldc.i4.s   100
     IL_002a:  stsfld     int32 TailcallVerify.Condition10::Result
     .try

src/tests/JIT/opt/Tailcall/TailcallVerifyWithPrefix.ilproj

@@ -15,5 +15,6 @@
     <Compile Include="TailcallVerifyWithPrefix.il" />
     <Compile Include="TailcallVerifyTransparentLibraryWithPrefix.il" />
     <Compile Include="TailcallVerifyVerifiableLibraryWithPrefix.il" />
+    <ProjectReference Include="$(TestLibraryProjectPath)" />
   </ItemGroup>
 </Project>