[concurrency] Change #isolated to mask out the TBI bits of the witness pointer of the implicit isolated any Actor pointer so we can do optimizations on TBI supporting platforms in the future.

include/swift/Basic/Features.def

@@ -541,6 +541,10 @@ SUPPRESSIBLE_EXPERIMENTAL_FEATURE(Lifetimes, true)
 /// Allow macro based aliases to be imported into Swift
 EXPERIMENTAL_FEATURE(ImportMacroAliases, true)
 
+/// Allow support for the nonisolated(nonsending) dynamic hop elim off of
+/// on platforms that do not support aarch64 tbi.
+EXPERIMENTAL_FEATURE(NonisolatedNonsendingDynamicHopElim, false)
+
 #undef EXPERIMENTAL_FEATURE_EXCLUDED_FROM_MODULE_INTERFACE
 #undef EXPERIMENTAL_FEATURE
 #undef UPCOMING_FEATURE

include/swift/Basic/LangOptions.h

@@ -670,6 +670,9 @@ namespace swift {
     bool RestrictNonProductionExperimentalFeatures = false;
 #endif
 
+    /// Set to true if we support AArch64TBI.
+    bool HasAArch64TBI = false;
+
     bool isConcurrencyModelTaskToThread() const {
       return ActiveConcurrencyModel == ConcurrencyModel::TaskToThread;
     }

include/swift/Frontend/Frontend.h

@@ -274,6 +274,9 @@ class CompilerInvocation {
   /// C++ stdlib is the default for the specified target.
   void computeCXXStdlibOptions();
 
+  /// Compute whether or not we support aarch64TBI
+  void computeAArch64TBIOptions();
+
   /// Computes the runtime resource path relative to the given Swift
   /// executable.
   static void computeRuntimeResourcePathFromExecutablePath(

include/swift/SIL/SILType.h

@@ -1025,6 +1025,10 @@ class SILType {
   /// Return '()'
   static SILType getEmptyTupleType(const ASTContext &C);
 
+  /// Return (elementTypes).
+  static SILType getTupleType(const ASTContext &ctx,
+                              ArrayRef<SILType> elementTypes);
+
   /// Get the type for opaque actor isolation values.
   static SILType getOpaqueIsolationType(const ASTContext &C);
 

lib/AST/FeatureSet.cpp

@@ -123,6 +123,7 @@ UNINTERESTING_FEATURE(SuppressedAssociatedTypes)
 UNINTERESTING_FEATURE(StructLetDestructuring)
 UNINTERESTING_FEATURE(MacrosOnImports)
 UNINTERESTING_FEATURE(NonisolatedNonsendingByDefault)
+UNINTERESTING_FEATURE(NonisolatedNonsendingDynamicHopElim)
 UNINTERESTING_FEATURE(KeyPathWithMethodMembers)
 UNINTERESTING_FEATURE(ImportMacroAliases)
 UNINTERESTING_FEATURE(NoExplicitNonIsolated)

lib/DriverTool/sil_opt_main.cpp

@@ -814,6 +814,8 @@ int sil_opt_main(ArrayRef<const char *> argv, void *MainAddr) {
   Invocation.getLangOptions().UnavailableDeclOptimizationMode =
       options.UnavailableDeclOptimization;
 
+  Invocation.computeAArch64TBIOptions();
+
   // Enable strict concurrency if we have the feature specified or if it was
   // specified via a command line option to sil-opt.
   if (Invocation.getLangOptions().hasFeature(Feature::StrictConcurrency)) {

lib/Frontend/CompilerInvocation.cpp

@@ -382,6 +382,13 @@ setBridgingHeaderFromFrontendOptions(ClangImporterOptions &ImporterOpts,
       FrontendOpts.InputsAndOutputs.getFilenameOfFirstInput();
 }
 
+void CompilerInvocation::computeAArch64TBIOptions() {
+  auto &LLVMArgs = getFrontendOptions().LLVMArgs;
+  auto aarch64_use_tbi =
+      std::find(LLVMArgs.begin(), LLVMArgs.end(), "-aarch64-use-tbi");
+  LangOpts.HasAArch64TBI = aarch64_use_tbi != LLVMArgs.end();
+}
+
 void CompilerInvocation::computeCXXStdlibOptions() {
   // The MSVC driver in Clang is not aware of the C++ stdlib, and currently
   // always assumes libstdc++, which is incorrect: the Microsoft stdlib is
@@ -4108,6 +4115,8 @@ bool CompilerInvocation::parseArgs(
   setIRGenOutputOptsFromFrontendOptions(IRGenOpts, FrontendOpts);
   setBridgingHeaderFromFrontendOptions(ClangImporterOpts, FrontendOpts);
   computeCXXStdlibOptions();
+  computeAArch64TBIOptions();
+
   if (LangOpts.hasFeature(Feature::Embedded)) {
     IRGenOpts.InternalizeAtLink = true;
     IRGenOpts.DisableLegacyTypeInfo = true;

lib/IRGen/GenThunk.cpp

@@ -137,6 +137,23 @@ void IRGenThunk::prepareArguments() {
 
   if (isWitnessMethod) {
     witnessMetadata.SelfWitnessTable = original.takeLast();
+
+    // Mask out the bottom bits when we dispatch specifically to
+    // Actor.unownedExecutor if we NonisolatedNonsendingDynamicHopElim is
+    // enabled and we are not on a platform that supports TBI.
+    auto &langOpts = IGF.getSILModule().getASTContext().LangOpts;
+    if (langOpts.hasFeature(Feature::NonisolatedNonsendingDynamicHopElim) &&
+        !langOpts.HasAArch64TBI &&
+        IGF.CurFn->getName() == "$sScA15unownedExecutorScevgTj") {
+      auto type = llvm::Type::getInt64Ty(IGF.CurFn->getContext());
+      auto pointerCast = IGF.Builder.CreateBitOrPointerCast(
+          witnessMetadata.SelfWitnessTable, type);
+      witnessMetadata.SelfWitnessTable = IGF.Builder.CreateBitOrPointerCast(
+          IGF.Builder.CreateBinOp(llvm::Instruction::BinaryOps::And,
+                                  pointerCast,
+                                  llvm::ConstantInt::get(IGF.IGM.IntPtrTy, -4)),
+          witnessMetadata.SelfWitnessTable->getType());
+    }
     witnessMetadata.SelfMetadata = original.takeLast();
   }
 

lib/SIL/IR/SILType.cpp

@@ -103,6 +103,21 @@ SILType SILType::getEmptyTupleType(const ASTContext &C) {
   return getPrimitiveObjectType(C.TheEmptyTupleType);
 }
 
+SILType SILType::getTupleType(const ASTContext &ctx,
+                              ArrayRef<SILType> elementTypes) {
+  if (elementTypes.empty())
+    return SILType::getEmptyTupleType(ctx);
+
+  SmallVector<TupleTypeElt, 8> tupleTypeElts;
+  auto category = elementTypes[0].getCategory();
+  for (auto type : elementTypes) {
+    assert(type.getCategory() == category && "Mismatched tuple elt categories");
+    tupleTypeElts.push_back(TupleTypeElt(type.getRawASTType()));
+  }
+  auto tupleType = TupleType::get(tupleTypeElts, ctx);
+  return SILType::getPrimitiveType(tupleType->getCanonicalType(), category);
+}
+
 SILType SILType::getSILTokenType(const ASTContext &C) {
   return getPrimitiveObjectType(C.TheSILTokenType);
 }

lib/SILGen/CMakeLists.txt

@@ -3,6 +3,7 @@ add_swift_host_library(swiftSILGen STATIC
   ArgumentSource.cpp
   Cleanup.cpp
   Condition.cpp
+  ConcurrencyUtils.cpp
   FormalEvaluation.cpp
   ManagedValue.cpp
   ResultPlan.cpp

lib/SILGen/ConcurrencyUtils.cpp

@@ -0,0 +1,163 @@
+//===--- ConcurrencyUtils.cpp ---------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2025 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+#include "ConcurrencyUtils.h"
+
+#include "RValue.h"
+#include "SILGenFunction.h"
+
+using namespace swift;
+using namespace swift::Lowering;
+
+// We could use a higher bit. But by reusing the first bit, we just save a
+// little bit of code.
+//
+// $valueToShift << (((sizeof(Word) - 1) << 3) + 4)
+//
+// Mathematicaly this is $valueToShift * 2**((sizeof(Word) - 1)*8 + 4)
+//
+// On 64 bit this is 60. This works since we want to use the bottom two bits of
+// the top nibble of the TBI bits.
+static SILValue getTBIBits(SILGenFunction &SGF, SILLocation loc,
+                           unsigned valueToShift = 1) {
+  auto &ctx = SGF.getASTContext();
+  auto silWordType = SILType::getBuiltinWordType(ctx);
+
+  auto id = ctx.getIdentifier(getBuiltinName(BuiltinValueKind::Sizeof));
+  auto *builtin = ::cast<FuncDecl>(getBuiltinValueDecl(ctx, id));
+  auto wordType = BuiltinIntegerType::getWordType(ctx)->getCanonicalType();
+  auto metatypeTy = SILType::getPrimitiveObjectType(CanMetatypeType::get(
+      wordType->getCanonicalType(), MetatypeRepresentation::Thin));
+  auto metatypeVal = SGF.B.createMetatype(loc, metatypeTy);
+
+  auto sizeOfWord =
+      SGF.B.createBuiltin(loc, id, silWordType,
+                          SubstitutionMap::get(builtin->getGenericSignature(),
+                                               ArrayRef<Type>{wordType},
+                                               LookUpConformanceInModule()),
+                          {metatypeVal});
+  auto one = SGF.B.createIntegerLiteral(loc, silWordType, 1);
+  auto three = SGF.B.createIntegerLiteral(loc, silWordType, 3);
+  auto four = SGF.B.createIntegerLiteral(loc, silWordType, 4);
+  auto valueToShiftLit =
+      SGF.B.createIntegerLiteral(loc, silWordType, valueToShift);
+
+  // sizeof(Word) - 1
+  auto sub = SGF.B.createBuiltinBinaryFunction(loc, "sub", silWordType,
+                                               silWordType, {sizeOfWord, one});
+  // (sizeof(Word) - 1) << 3
+  auto innerShift = SGF.B.createBuiltinBinaryFunction(
+      loc, "shl", silWordType, silWordType, {sub, three});
+  // ((sizeof(Word) - 1) << 3) + 4
+  auto innerShiftOffset = SGF.B.createBuiltinBinaryFunction(
+      loc, "add", silWordType, silWordType, {innerShift, four});
+  auto outerShift =
+      SGF.B.createBuiltinBinaryFunction(loc, "shl", silWordType, silWordType,
+                                        {valueToShiftLit, innerShiftOffset});
+  return outerShift;
+}
+
+/// Construct the TBI mask in a platform independent way that works on all
+/// platforms.
+///
+/// We compute:
+///
+///   mask = (0x3 << (((sizeof(Word) - 1) << 3) + 4)) ^ -1
+static SILValue getTBIClearMask(SILGenFunction &SGF, SILLocation loc) {
+  auto &ctx = SGF.getASTContext();
+  auto silWordType = SILType::getBuiltinWordType(ctx);
+  auto negBits = SGF.B.createIntegerLiteral(loc, silWordType, -1);
+
+  return SGF.B.createBuiltinBinaryFunction(loc, "xor", silWordType, silWordType,
+                                           {getTBIBits(SGF, loc, 3), negBits});
+}
+
+static ManagedValue
+transformTupleElts(SILGenFunction &SGF, SILLocation loc, ManagedValue mv,
+                   llvm::function_ref<SILValue(DestructureTupleInst *)> func) {
+  auto &ctx = SGF.getASTContext();
+  auto silWordType = SILType::getBuiltinWordType(ctx);
+  auto tupleType =
+      SILType::getTupleType(SGF.getASTContext(), {silWordType, silWordType});
+  auto cast =
+      SGF.B.createUncheckedBitCast(loc, mv, tupleType).getUnmanagedValue();
+  auto *destructure = SGF.B.createDestructureTuple(loc, cast);
+  SILValue reformedValue = func(destructure);
+  auto reformedPointer =
+      ManagedValue::forBorrowedRValue(SGF.B.createUncheckedOwnershipConversion(
+          loc,
+          SGF.B.createUncheckedReinterpretCast(loc, reformedValue,
+                                               mv.getType()),
+          OwnershipKind::Guaranteed));
+  return SGF.B.createMarkDependence(loc, reformedPointer, mv,
+                                    MarkDependenceKind::NonEscaping);
+}
+
+ManagedValue swift::Lowering::clearImplicitIsolatedActorBits(
+    SILGenFunction &SGF, SILLocation loc, ManagedValue isolatedMV) {
+  auto &ctx = SGF.getASTContext();
+  if (!ctx.LangOpts.hasFeature(Feature::NonisolatedNonsendingDynamicHopElim) &&
+      !ctx.LangOpts.HasAArch64TBI) {
+    return isolatedMV;
+  }
+
+  return transformTupleElts(
+      SGF, loc, isolatedMV, [&](DestructureTupleInst *dti) {
+        auto silWordType = SILType::getBuiltinWordType(ctx);
+        SILValue bitMask =
+            ctx.LangOpts.HasAArch64TBI
+                ? getTBIClearMask(SGF, loc)
+                : SGF.B.createIntegerLiteral(loc, silWordType, -4);
+
+        auto result = SGF.B.createBuiltinBinaryFunction(
+            loc, "and", silWordType, silWordType, {dti->getResult(1), bitMask});
+        return SGF.B.createTuple(loc, {dti->getResult(0), result});
+      });
+}
+
+RValue
+swift::Lowering::clearImplicitIsolatedActorBits(SILGenFunction &SGF, Expr *expr,
+                                                ManagedValue isolatedMV) {
+  return RValue(
+      SGF, expr,
+      clearImplicitIsolatedActorBits(SGF, SILLocation(expr), isolatedMV));
+}
+
+ManagedValue swift::Lowering::setImplicitIsolatedActorBits(
+    SILGenFunction &SGF, SILLocation loc, ManagedValue isolatedMV) {
+  auto &ctx = SGF.getASTContext();
+  if (!ctx.LangOpts.hasFeature(Feature::NonisolatedNonsendingDynamicHopElim) &&
+      !ctx.LangOpts.HasAArch64TBI) {
+    return isolatedMV;
+  }
+
+  return transformTupleElts(
+      SGF, loc, isolatedMV, [&](DestructureTupleInst *dti) {
+        auto silWordType = SILType::getBuiltinWordType(ctx);
+        SILValue bitMask =
+            ctx.LangOpts.HasAArch64TBI
+                ? getTBIBits(SGF, loc)
+                : SGF.B.createIntegerLiteral(loc, silWordType, 1);
+
+        auto result = SGF.B.createBuiltinBinaryFunction(
+            loc, "or", silWordType, silWordType, {dti->getResult(1), bitMask});
+        return SGF.B.createTuple(loc, {dti->getResult(0), result});
+      });
+}
+
+RValue swift::Lowering::setImplicitIsolatedActorBits(SILGenFunction &SGF,
+                                                     Expr *expr,
+                                                     ManagedValue isolatedMV) {
+  return RValue(
+      SGF, expr,
+      setImplicitIsolatedActorBits(SGF, SILLocation(expr), isolatedMV));
+}

lib/SILGen/ConcurrencyUtils.h

@@ -0,0 +1,50 @@
+//===--- ConcurrencyUtils.h -----------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2025 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SWIFT_SILGEN_CONCURRENCYUTILS_H
+#define SWIFT_SILGEN_CONCURRENCYUTILS_H
+
+namespace swift {
+
+class SILLocation;
+class Expr;
+
+namespace Lowering {
+
+class SILGenFunction;
+class RValue;
+class ManagedValue;
+
+/// Clear the TBI bits if AArch64HasTBI is set. Otherwise clear the low tagged
+/// bits.
+///
+/// \param expr - the expression which yielded this r-value; its type
+///   will become the substituted formal type of this r-value
+/// \param implicitIsolatedActor should be an Optional<any Actor>.
+RValue clearImplicitIsolatedActorBits(SILGenFunction &SGF, Expr *expr,
+                                      ManagedValue implicitIsolatedActor);
+
+ManagedValue clearImplicitIsolatedActorBits(SILGenFunction &SGF,
+                                            SILLocation loc,
+                                            ManagedValue implicitIsolatedActor);
+
+RValue setImplicitIsolatedActorBits(SILGenFunction &SGF, Expr *expr,
+                                    ManagedValue implicitIsolatedActor);
+
+ManagedValue setImplicitIsolatedActorBits(SILGenFunction &SGF, SILLocation loc,
+                                          ManagedValue implicitIsolatedActor);
+
+} // namespace Lowering
+
+} // namespace swift
+
+#endif

lib/SILGen/SILGenApply.cpp

@@ -13,6 +13,7 @@
 #include "ArgumentScope.h"
 #include "ArgumentSource.h"
 #include "Callee.h"
+#include "ConcurrencyUtils.h"
 #include "Conversion.h"
 #include "ExecutorBreadcrumb.h"
 #include "FormalEvaluation.h"
@@ -5861,8 +5862,12 @@ ApplyOptions CallEmission::emitArgumentsForNormalApply(
     args.push_back({});
     // NOTE: Even though this calls emitActorInstanceIsolation, this also
     // handles glboal actor isolated cases.
-    args.back().push_back(SGF.emitActorInstanceIsolation(
-        callSite->Loc, executor, executor.getType().getASTType()));
+    auto erasedActor =
+        SGF.emitActorInstanceIsolation(callSite->Loc, executor,
+                                       executor.getType().getASTType())
+            .borrow(SGF, callSite->Loc);
+    args.back().push_back(
+        clearImplicitIsolatedActorBits(SGF, callSite->Loc, erasedActor));
   }
 
   uncurriedLoc = callSite->Loc;

lib/SILGen/SILGenConcurrency.cpp

@@ -27,7 +27,7 @@ using namespace Lowering;
 
 static void setExpectedExecutorForGeneric(SILGenFunction &SGF) {
   auto loc = RegularLocation::getAutoGeneratedLocation(SGF.F.getLocation());
-  SGF.ExpectedExecutor.set(SGF.emitGenericExecutor(loc));
+  SGF.ExpectedExecutor.set(SGF.emitNonIsolatedIsolation(loc).getValue());
 }
 
 static void setExpectedExecutorForGlobalActor(SILGenFunction &SGF,
@@ -284,8 +284,9 @@ void SILGenFunction::emitConstructorExpectedExecutorProlog() {
       loc.markAsPrologue();
       loc = loc.asAutoGenerated();
 
-      auto initialExecutor = emitGenericExecutor(loc);
-      B.createHopToExecutor(loc, initialExecutor, /*mandatory*/ false);
+      auto initialExecutor = emitNonIsolatedIsolation(loc);
+      B.createHopToExecutor(loc, initialExecutor.getValue(),
+                            /*mandatory*/ false);
       return;
     }
   }