Utils: Inhibit load/store folding through phis for llvm.protected.field.ptr.

clang/lib/CodeGen/CGPointerAuth.cpp

@@ -440,9 +440,9 @@ CodeGenModule::getConstantSignedPointer(llvm::Constant *Pointer, unsigned Key,
     IntegerDiscriminator = llvm::ConstantInt::get(Int64Ty, 0);
   }
 
-  return llvm::ConstantPtrAuth::get(Pointer,
-                                    llvm::ConstantInt::get(Int32Ty, Key),
-                                    IntegerDiscriminator, AddressDiscriminator);
+  return llvm::ConstantPtrAuth::get(
+      Pointer, llvm::ConstantInt::get(Int32Ty, Key), IntegerDiscriminator,
+      AddressDiscriminator, llvm::Constant::getNullValue(UnqualPtrTy));
 }
 
 /// Does a given PointerAuthScheme require us to sign a value

lld/ELF/Arch/AArch64.cpp

@@ -114,6 +114,7 @@ AArch64::AArch64(Ctx &ctx) : TargetInfo(ctx) {
   copyRel = R_AARCH64_COPY;
   relativeRel = R_AARCH64_RELATIVE;
   iRelativeRel = R_AARCH64_IRELATIVE;
+  iRelSymbolicRel = R_AARCH64_FUNCINIT64;
   gotRel = R_AARCH64_GLOB_DAT;
   pltRel = R_AARCH64_JUMP_SLOT;
   symbolicRel = R_AARCH64_ABS64;
@@ -137,6 +138,7 @@ RelExpr AArch64::getRelExpr(RelType type, const Symbol &s,
   case R_AARCH64_ABS16:
   case R_AARCH64_ABS32:
   case R_AARCH64_ABS64:
+  case R_AARCH64_FUNCINIT64:
   case R_AARCH64_ADD_ABS_LO12_NC:
   case R_AARCH64_LDST128_ABS_LO12_NC:
   case R_AARCH64_LDST16_ABS_LO12_NC:
@@ -154,6 +156,12 @@ RelExpr AArch64::getRelExpr(RelType type, const Symbol &s,
   case R_AARCH64_MOVW_UABS_G2_NC:
   case R_AARCH64_MOVW_UABS_G3:
     return R_ABS;
+  case R_AARCH64_PATCHINST:
+    if (!isAbsolute(s))
+      Err(ctx) << getErrorLoc(ctx, loc)
+               << "R_AARCH64_PATCHINST relocation against non-absolute symbol "
+               << &s;
+    return R_ABS;
   case R_AARCH64_AUTH_ABS64:
     return RE_AARCH64_AUTH;
   case R_AARCH64_TLSDESC_ADR_PAGE21:
@@ -261,7 +269,8 @@ bool AArch64::usesOnlyLowPageBits(RelType type) const {
 }
 
 RelType AArch64::getDynRel(RelType type) const {
-  if (type == R_AARCH64_ABS64 || type == R_AARCH64_AUTH_ABS64)
+  if (type == R_AARCH64_ABS64 || type == R_AARCH64_AUTH_ABS64 ||
+      type == R_AARCH64_FUNCINIT64)
     return type;
   return R_AARCH64_NONE;
 }
@@ -506,6 +515,12 @@ void AArch64::relocate(uint8_t *loc, const Relocation &rel,
     checkIntUInt(ctx, loc, val, 32, rel);
     write32(ctx, loc, val);
     break;
+  case R_AARCH64_PATCHINST:
+    if (!rel.sym->isUndefined()) {
+      checkUInt(ctx, loc, val, 32, rel);
+      write32le(loc, val);
+    }
+    break;
   case R_AARCH64_PLT32:
   case R_AARCH64_GOTPCREL32:
     checkInt(ctx, loc, val, 32, rel);

lld/ELF/Relocations.cpp

@@ -176,7 +176,7 @@ static RelType getMipsPairType(RelType type, bool isLocal) {
 
 // True if non-preemptable symbol always has the same value regardless of where
 // the DSO is loaded.
-static bool isAbsolute(const Symbol &sym) {
+bool elf::isAbsolute(const Symbol &sym) {
   if (sym.isUndefined())
     return true;
   if (const auto *dr = dyn_cast<Defined>(&sym))
@@ -989,8 +989,8 @@ bool RelocationScanner::isStaticLinkTimeConstant(RelExpr e, RelType type,
   // only the low bits are used.
   if (e == R_GOT || e == R_PLT)
     return ctx.target->usesOnlyLowPageBits(type) || !ctx.arg.isPic;
-  // R_AARCH64_AUTH_ABS64 requires a dynamic relocation.
-  if (e == RE_AARCH64_AUTH)
+  // R_AARCH64_AUTH_ABS64 and iRelSymbolicRel require a dynamic relocation.
+  if (e == RE_AARCH64_AUTH || type == ctx.target->iRelSymbolicRel)
     return false;
 
   // The behavior of an undefined weak reference is implementation defined.
@@ -1163,6 +1163,24 @@ void RelocationScanner::processAux(RelExpr expr, RelType type, uint64_t offset,
         }
         return;
       }
+      if (LLVM_UNLIKELY(type == ctx.target->iRelSymbolicRel)) {
+        if (sym.isPreemptible) {
+          auto diag = Err(ctx);
+          diag << "relocation " << type
+               << " cannot be used against preemptible symbol '" << &sym << "'";
+          printLocation(diag, *sec, sym, offset);
+        } else if (isIfunc) {
+          auto diag = Err(ctx);
+          diag << "relocation " << type
+               << " cannot be used against ifunc symbol '" << &sym << "'";
+          printLocation(diag, *sec, sym, offset);
+        } else {
+          part.relaDyn->addReloc({ctx.target->iRelativeRel, sec, offset,
+                                  DynamicReloc::AddendOnlyWithTargetVA, sym,
+                                  addend, R_ABS});
+          return;
+        }
+      }
       part.relaDyn->addSymbolReloc(rel, *sec, offset, sym, addend, type);
 
       // MIPS ABI turns using of GOT and dynamic relocations inside out.

lld/ELF/Relocations.h

@@ -165,6 +165,8 @@ void addGotEntry(Ctx &ctx, Symbol &sym);
 void hexagonTLSSymbolUpdate(Ctx &ctx);
 bool hexagonNeedsTLSSymbol(ArrayRef<OutputSection *> outputSections);
 
+bool isAbsolute(const Symbol &sym);
+
 class ThunkSection;
 class Thunk;
 class InputSectionDescription;

lld/ELF/Target.h

@@ -131,6 +131,7 @@ class TargetInfo {
   RelType relativeRel = 0;
   RelType iRelativeRel = 0;
   RelType symbolicRel = 0;
+  RelType iRelSymbolicRel = 0;
   RelType tlsDescRel = 0;
   RelType tlsGotRel = 0;
   RelType tlsModuleIndexRel = 0;

lld/test/ELF/aarch64-funcinit64-invalid.s

@@ -0,0 +1,18 @@
+# REQUIRES: aarch64
+
+# RUN: llvm-mc -filetype=obj -triple=aarch64 %s -o %t.o
+# RUN: not ld.lld %t.o -o %t 2>&1 | FileCheck --check-prefix=ERR %s
+
+.rodata
+# ERR: relocation R_AARCH64_FUNCINIT64 cannot be used against local symbol
+.8byte func@FUNCINIT
+
+.data
+# ERR: relocation R_AARCH64_FUNCINIT64 cannot be used against ifunc symbol 'ifunc'
+.8byte ifunc@FUNCINIT
+
+.text
+func:
+.type ifunc, @gnu_indirect_function
+ifunc:
+ret

lld/test/ELF/aarch64-funcinit64.s

@@ -0,0 +1,19 @@
+# REQUIRES: aarch64
+
+# RUN: llvm-mc -filetype=obj -triple=aarch64 %s -o %t.o
+# RUN: ld.lld %t.o -o %t
+# RUN: llvm-readelf -s -r %t | FileCheck %s
+# RUN: ld.lld %t.o -o %t -pie
+# RUN: llvm-readelf -s -r %t | FileCheck %s
+# RUN: not ld.lld %t.o -o %t -shared 2>&1 | FileCheck --check-prefix=ERR %s
+
+.data
+# CHECK: R_AARCH64_IRELATIVE [[FOO:[0-9a-f]*]]
+# ERR: relocation R_AARCH64_FUNCINIT64 cannot be used against preemptible symbol 'foo'
+.8byte foo@FUNCINIT
+
+.text
+# CHECK: {{0*}}[[FOO]] {{.*}} foo
+.globl foo
+foo:
+ret

lld/test/ELF/aarch64-patchinst.s

@@ -0,0 +1,87 @@
+# RUN: rm -rf %t && split-file %s %t
+# RUN: llvm-mc -filetype=obj -triple=aarch64 %t/use.s -o %t/use-le.o
+# RUN: llvm-mc -filetype=obj -triple=aarch64 %t/def.s -o %t/def-le.o
+# RUN: llvm-mc -filetype=obj -triple=aarch64 %t/rel.s -o %t/rel-le.o
+
+## Deactivation symbol used without being defined: instruction emitted as usual.
+# RUN: ld.lld -o %t/undef-le %t/use-le.o --emit-relocs
+# RUN: llvm-objdump -r %t/undef-le | FileCheck --check-prefix=RELOC %s
+# RUN: llvm-objdump -d %t/undef-le | FileCheck --check-prefix=UNDEF %s
+# RUN: ld.lld -pie -o %t/undef-le %t/use-le.o --emit-relocs
+# RUN: llvm-objdump -r %t/undef-le | FileCheck --check-prefix=RELOC %s
+# RUN: llvm-objdump -d %t/undef-le | FileCheck --check-prefix=UNDEF %s
+
+## Deactivation symbol defined: instructions overwritten with NOPs.
+# RUN: ld.lld -o %t/def-le %t/use-le.o %t/def-le.o --emit-relocs
+# RUN: llvm-objdump -r %t/def-le | FileCheck --check-prefix=RELOC %s
+# RUN: llvm-objdump -d %t/def-le | FileCheck --check-prefix=DEF %s
+# RUN: ld.lld -pie -o %t/def-le %t/use-le.o %t/def-le.o --emit-relocs
+# RUN: llvm-objdump -r %t/def-le | FileCheck --check-prefix=RELOC %s
+# RUN: llvm-objdump -d %t/def-le | FileCheck --check-prefix=DEF %s
+
+## Relocation pointing to a non-SHN_UNDEF non-SHN_ABS symbol is an error.
+# RUN: not ld.lld -o %t/rel-le %t/use-le.o %t/rel-le.o 2>&1 | FileCheck --check-prefix=ERROR %s
+# RUN: not ld.lld -pie -o %t/rel-le %t/use-le.o %t/rel-le.o 2>&1 | FileCheck --check-prefix=ERROR %s
+
+## Behavior unchanged by endianness: relocation always written as little endian.
+# RUN: llvm-mc -filetype=obj -triple=aarch64_be %t/use.s -o %t/use-be.o
+# RUN: llvm-mc -filetype=obj -triple=aarch64_be %t/def.s -o %t/def-be.o
+# RUN: llvm-mc -filetype=obj -triple=aarch64_be %t/rel.s -o %t/rel-be.o
+# RUN: ld.lld -o %t/undef-be %t/use-be.o --emit-relocs
+# RUN: llvm-objdump -r %t/undef-be | FileCheck --check-prefix=RELOC %s
+# RUN: llvm-objdump -d %t/undef-be | FileCheck --check-prefix=UNDEF %s
+# RUN: ld.lld -pie -o %t/undef-be %t/use-be.o --emit-relocs
+# RUN: llvm-objdump -r %t/undef-be | FileCheck --check-prefix=RELOC %s
+# RUN: llvm-objdump -d %t/undef-be | FileCheck --check-prefix=UNDEF %s
+# RUN: ld.lld -o %t/def-be %t/use-be.o %t/def-be.o --emit-relocs
+# RUN: llvm-objdump -r %t/def-be | FileCheck --check-prefix=RELOC %s
+# RUN: llvm-objdump -d %t/def-be | FileCheck --check-prefix=DEF %s
+# RUN: ld.lld -pie -o %t/def-be %t/use-be.o %t/def-be.o --emit-relocs
+# RUN: llvm-objdump -r %t/def-be | FileCheck --check-prefix=RELOC %s
+# RUN: llvm-objdump -d %t/def-be | FileCheck --check-prefix=DEF %s
+# RUN: not ld.lld -o %t/rel-be %t/use-be.o %t/rel-be.o 2>&1 | FileCheck --check-prefix=ERROR %s
+# RUN: not ld.lld -pie -o %t/rel-be %t/use-be.o %t/rel-be.o 2>&1 | FileCheck --check-prefix=ERROR %s
+
+# RELOC:      R_AARCH64_JUMP26
+# RELOC-NEXT: R_AARCH64_PATCHINST ds
+# RELOC-NEXT: R_AARCH64_PATCHINST ds
+# RELOC-NEXT: R_AARCH64_PATCHINST ds0+0xd503201f
+
+#--- use.s
+.weak ds
+.weak ds0
+# This instruction has a single relocation: the DS relocation.
+# UNDEF: add x0, x1, x2
+# DEF: nop
+# ERROR: R_AARCH64_PATCHINST relocation against non-absolute symbol ds
+.reloc ., R_AARCH64_PATCHINST, ds
+add x0, x1, x2
+# This instruction has two relocations: the DS relocation and the JUMP26 to f1.
+# Make sure that the DS relocation takes precedence.
+.reloc ., R_AARCH64_PATCHINST, ds
+# UNDEF: b {{.*}} <f1>
+# DEF: nop
+# ERROR: R_AARCH64_PATCHINST relocation against non-absolute symbol ds
+b f1
+# Alternative representation: instruction opcode stored in addend.
+# UNDEF: add x3, x4, x5
+# DEF: nop
+# ERROR: R_AARCH64_PATCHINST relocation against non-absolute symbol ds0
+.reloc ., R_AARCH64_PATCHINST, ds0 + 0xd503201f
+add x3, x4, x5
+
+.section .text.f1,"ax",@progbits
+f1:
+ret
+
+#--- def.s
+.globl ds
+ds = 0xd503201f
+.globl ds0
+ds0 = 0
+
+#--- rel.s
+.globl ds
+ds:
+.globl ds0
+ds0:

llvm/docs/LangRef.rst

@@ -3090,6 +3090,21 @@ A "convergencectrl" operand bundle is only valid on a ``convergent`` operation.
 When present, the operand bundle must contain exactly one value of token type.
 See the :doc:`ConvergentOperations` document for details.
 
+Deactivation Symbol Operand Bundles
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+A ``"deactivation-symbol"`` operand bundle is valid on the following
+instructions (AArch64 only):
+
+- Call to a normal function with ``notail`` attribute.
+- Call to ``llvm.ptrauth.sign`` or ``llvm.ptrauth.auth`` intrinsics.
+
+This operand bundle specifies that if the deactivation symbol is defined
+to a valid value for the target, the marked instruction will return the
+value of its first argument instead of calling the specified function
+or intrinsic. This is achieved with ``PATCHINST`` relocations on the
+target instructions (see the AArch64 psABI for details).
+
 .. _moduleasm:
 
 Module-Level Inline Assembly
@@ -31146,3 +31161,57 @@ This intrinsic is assumed to execute in the default :ref:`floating-point
 environment <floatenv>` *except* for the rounding mode.
 This intrinsic is not supported on all targets. Some targets may not support
 all rounding modes.
+
+'``llvm.protected.field.ptr``' Intrinsic
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Syntax:
+"""""""
+
+::
+
+      declare ptr @llvm.protected.field.ptr(ptr ptr, i64 disc, i1 use_hw_encoding)
+
+Overview:
+"""""""""
+
+The '``llvm.protected.field.ptr``' intrinsic returns a pointer to the
+storage location of a pointer that has special properties as described
+below.
+
+Arguments:
+""""""""""
+
+The first argument is the pointer specifying the location to store the
+pointer. The second argument is the discriminator, which is used as an
+input for the pointer encoding. The third argument specifies whether to
+use a target-specific mechanism to encode the pointer.
+
+Semantics:
+""""""""""
+
+This intrinsic returns a pointer which may be used to store a
+pointer at the specified address that is encoded using the specified
+discriminator. Stores via the pointer will cause the stored pointer to be
+blended with the second argument before being stored. The blend operation
+shall be either a weak but cheap and target-independent operation (if
+the third argument is 0) or a stronger target-specific operation (if the
+third argument is 1). When loading from the pointer, the inverse operation
+is done on the loaded pointer after it is loaded. Specifically, when the
+third argument is 1, the pointer is signed (using pointer authentication
+instructions or emulated PAC if not supported by the hardware) using
+the struct address before being stored, and authenticated after being
+loaded. Note that it is currently unsupported to have the third argument
+be 1 on targets other than AArch64. When the third argument is 0, it is
+rotated left by 16 bits and the discriminator is subtracted before being
+stored, and the discriminator is added and the pointer is rotated right
+by 16 bits after being loaded.
+
+If the pointer is used otherwise than for loading or storing (e.g. its
+address escapes), that will disable all blending operations using
+the deactivation symbol specified in the intrinsic's operand bundle.
+The deactivation symbol operand bundle is copied onto any sign and auth
+intrinsics that this intrinsic is lowered into. The intent is that the
+deactivation symbol represents a field identifier.
+
+This intrinsic is used to implement structure protection.

llvm/include/llvm/BinaryFormat/ELFRelocs/AArch64.def

@@ -61,6 +61,8 @@ ELF_RELOC(R_AARCH64_LD64_GOT_LO12_NC,                0x138)
 ELF_RELOC(R_AARCH64_LD64_GOTPAGE_LO15,               0x139)
 ELF_RELOC(R_AARCH64_PLT32,                           0x13a)
 ELF_RELOC(R_AARCH64_GOTPCREL32,                      0x13b)
+ELF_RELOC(R_AARCH64_PATCHINST,                       0x13c)
+ELF_RELOC(R_AARCH64_FUNCINIT64,                      0x13d)
 // General dynamic TLS relocations
 ELF_RELOC(R_AARCH64_TLSGD_ADR_PREL21,                0x200)
 ELF_RELOC(R_AARCH64_TLSGD_ADR_PAGE21,                0x201)

llvm/include/llvm/Bitcode/LLVMBitCodes.h

@@ -437,6 +437,8 @@ enum ConstantsCodes {
   CST_CODE_CE_GEP_WITH_INRANGE = 31,  // [opty, flags, range, n x operands]
   CST_CODE_CE_GEP = 32,               // [opty, flags, n x operands]
   CST_CODE_PTRAUTH = 33,              // [ptr, key, disc, addrdisc]
+  CST_CODE_PTRAUTH2 = 34,             // [ptr, key, disc, addrdisc,
+                                      //  deactivation_symbol]
 };
 
 /// CastOpcodes - These are values used in the bitcode files to encode which

llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h

@@ -162,6 +162,8 @@ class LLVM_ABI CallLowering {
 
     /// True if this call results in convergent operations.
     bool IsConvergent = true;
+
+    GlobalValue *DeactivationSymbol = nullptr;
   };
 
   /// Argument handling is mostly uniform between the four places that

llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h

@@ -56,6 +56,7 @@ struct MachineIRBuilderState {
   MDNode *PCSections = nullptr;
   /// MMRA Metadata to be set on any instruction we create.
   MDNode *MMRA = nullptr;
+  Value *DS = nullptr;
 
   /// \name Fields describing the insertion point.
   /// @{
@@ -369,6 +370,7 @@ class LLVM_ABI MachineIRBuilder {
     State.II = MI.getIterator();
     setPCSections(MI.getPCSections());
     setMMRAMetadata(MI.getMMRAMetadata());
+    setDeactivationSymbol(MI.getDeactivationSymbol());
   }
   /// @}
 
@@ -405,6 +407,9 @@ class LLVM_ABI MachineIRBuilder {
   /// Set the PC sections metadata to \p MD for all the next build instructions.
   void setMMRAMetadata(MDNode *MMRA) { State.MMRA = MMRA; }
 
+  Value *getDeactivationSymbol() { return State.DS; }
+  void setDeactivationSymbol(Value *DS) { State.DS = DS; }
+
   /// Get the current instruction's MMRA metadata.
   MDNode *getMMRAMetadata() { return State.MMRA; }
 

llvm/include/llvm/CodeGen/ISDOpcodes.h

@@ -1563,6 +1563,10 @@ enum NodeType {
   // Outputs: Output Chain
   CLEAR_CACHE,
 
+  // Untyped node storing deactivation symbol reference
+  // (DeactivationSymbolSDNode).
+  DEACTIVATION_SYMBOL,
+
   /// BUILTIN_OP_END - This must be the last enum value in this list.
   /// The target-specific pre-isel opcode values start here.
   BUILTIN_OP_END