introduce @_manualOwnership performance attribute

SwiftCompilerSources/Sources/SIL/Function.swift

@@ -250,6 +250,7 @@ final public class Function : CustomStringConvertible, HasShortDescription, Hash
     case noRuntime
     case noExistentials
     case noObjCRuntime
+    case manualOwnership
   }
 
   public var performanceConstraints: PerformanceConstraints {
@@ -260,6 +261,7 @@ final public class Function : CustomStringConvertible, HasShortDescription, Hash
       case .NoRuntime: return .noRuntime
       case .NoExistentials: return .noExistentials
       case .NoObjCBridging: return .noObjCRuntime
+      case .ManualOwnership: return .manualOwnership
       default: fatalError("unknown performance constraint")
     }
   }

include/swift/AST/DeclAttr.def

@@ -811,7 +811,10 @@ SIMPLE_DECL_ATTR(_noObjCBridging, NoObjCBridging,
   UserInaccessible | ABIStableToAdd | ABIStableToRemove | APIStableToAdd | APIStableToRemove | ForbiddenInABIAttr,
   155)
 
-// Unused '156': Used to be `_distributedThunkTarget` but completed implementation in Swift 6.0 does not need it after all
+SIMPLE_DECL_ATTR(_manualOwnership, ManualOwnership,
+  OnAbstractFunction | OnSubscript,
+  UserInaccessible | ABIStableToAdd | ABIStableToRemove | APIStableToAdd | APIStableToRemove | ForbiddenInABIAttr,
+  156)
 
 DECL_ATTR(_allowFeatureSuppression, AllowFeatureSuppression,
   OnAnyDecl,

include/swift/AST/DiagnosticsSIL.def

@@ -428,6 +428,8 @@ ERROR(wrong_linkage_for_serialized_function,none,
       "function has wrong linkage to be called from %0", (StringRef))
 NOTE(performance_called_from,none,
       "called from here", ())
+ERROR(manualownership_copy,none,
+      "explicit 'copy' required here", ())
 
 // 'transparent' diagnostics
 ERROR(circular_transparent,none,

include/swift/AST/DiagnosticsSema.def

@@ -2100,6 +2100,10 @@ ERROR(attr_static_exclusive_only_mutating,none,
 ERROR(attr_static_exclusive_no_setters,none,
       "variable of type %0 must not have a setter", (Type))
 
+// @_manualOwnership
+ERROR(attr_manual_ownership_experimental,none,
+      "'@_manualOwnership' requires '-enable-experimental-feature ManualOwnership'", ())
+
 // @extractConstantsFromMembers
 ERROR(attr_extractConstantsFromMembers_experimental,none,
       "'@extractConstantsFromMembers' requires "

include/swift/AST/Expr.h

@@ -2292,7 +2292,7 @@ class CopyExpr final : public Expr {
 /// getSemanticsProvidingExpr() looks through this because it doesn't
 /// provide the value and only very specific clients care where the
 /// 'borrow' was written.
-class BorrowExpr final : public IdentityExpr {
+class BorrowExpr final : public IdentityExpr { // FIXME: this should be a ImplicitConversionExpr like LoadExpr
   SourceLoc BorrowLoc;
 
 public:

include/swift/Basic/Features.def

@@ -426,6 +426,9 @@ EXPERIMENTAL_FEATURE(LifetimeDependenceMutableAccessors, true)
 /// Enable the `@_staticExclusiveOnly` attribute.
 EXPERIMENTAL_FEATURE(StaticExclusiveOnly, true)
 
+/// Enable the `@_manualOwnership` attribute.
+EXPERIMENTAL_FEATURE(ManualOwnership, false)
+
 /// Enable the @extractConstantsFromMembers attribute.
 EXPERIMENTAL_FEATURE(ExtractConstantsFromMembers, false)
 

include/swift/SIL/SILBridging.h

@@ -434,7 +434,8 @@ struct BridgedFunction {
     NoLocks = 2,
     NoRuntime = 3,
     NoExistentials = 4,
-    NoObjCBridging = 5
+    NoObjCBridging = 5,
+    ManualOwnership = 6,
   };
 
   enum class InlineStrategy {

include/swift/SIL/SILBuilder.h

@@ -284,6 +284,14 @@ class SILBuilder {
     return false;
   }
 
+  /// If we have a SILFunction, return true if it has a ManualOwnership
+  /// PerformanceConstraint, which corresponds to an attribute on the FuncDecl.
+  bool hasManualOwnershipAttr() const {
+    if (F)
+      return F->getPerfConstraints() == PerformanceConstraints::ManualOwnership;
+    return false;
+  }
+
   //===--------------------------------------------------------------------===//
   // Insertion Point Management
   //===--------------------------------------------------------------------===//

include/swift/SIL/SILCloner.h

@@ -1701,6 +1701,16 @@ template<typename ImplClass>
 void
 SILCloner<ImplClass>::visitCopyAddrInst(CopyAddrInst *Inst) {
   getBuilder().setCurrentDebugScope(getOpScope(Inst->getDebugScope()));
+  // When cloning into a function using manual ownership, convert to explicit
+  // copies, in order to preserve the local nature of the perf constraint.
+  if (getBuilder().hasManualOwnershipAttr() && getBuilder().hasOwnership()) {
+    recordClonedInstruction(
+      Inst, getBuilder().createExplicitCopyAddr(
+                getOpLocation(Inst->getLoc()), getOpValue(Inst->getSrc()),
+                getOpValue(Inst->getDest()), Inst->isTakeOfSrc(),
+                Inst->isInitializationOfDest()));
+    return;
+  }
   recordClonedInstruction(
       Inst, getBuilder().createCopyAddr(
                 getOpLocation(Inst->getLoc()), getOpValue(Inst->getSrc()),
@@ -2095,6 +2105,15 @@ void SILCloner<ImplClass>::visitCopyValueInst(CopyValueInst *Inst) {
     return recordFoldedValue(Inst, newValue);
   }
 
+  // When cloning into a function using manual ownership, convert to explicit
+  // copies, in order to preserve the local nature of the perf constraint.
+  if (getBuilder().hasManualOwnershipAttr() && getBuilder().hasOwnership()) {
+    recordClonedInstruction(
+      Inst, getBuilder().createExplicitCopyValue(getOpLocation(Inst->getLoc()),
+                                         getOpValue(Inst->getOperand())));
+    return;
+  }
+
   recordClonedInstruction(
       Inst, getBuilder().createCopyValue(getOpLocation(Inst->getLoc()),
                                          getOpValue(Inst->getOperand())));

include/swift/SIL/SILFunction.h

@@ -92,7 +92,8 @@ enum class PerformanceConstraints : uint8_t {
   NoLocks = 2,
   NoRuntime = 3,
   NoExistentials = 4,
-  NoObjCBridging = 5
+  NoObjCBridging = 5,
+  ManualOwnership = 6,
 };
 
 class SILSpecializeAttr final {

lib/AST/ASTDumper.cpp

@@ -5000,6 +5000,7 @@ class PrintAttribute : public AttributeVisitor<PrintAttribute, void, Label>,
   TRIVIAL_ATTR_PRINTER(NoLocks, no_locks)
   TRIVIAL_ATTR_PRINTER(NoMetadata, no_metadata)
   TRIVIAL_ATTR_PRINTER(NoObjCBridging, no_objc_bridging)
+  TRIVIAL_ATTR_PRINTER(ManualOwnership, manual_ownership)
   TRIVIAL_ATTR_PRINTER(NoRuntime, no_runtime)
   TRIVIAL_ATTR_PRINTER(NonEphemeral, non_ephemeral)
   TRIVIAL_ATTR_PRINTER(NonEscapable, non_escapable)

lib/AST/ASTVerifier.cpp

@@ -2487,6 +2487,19 @@ class Verifier : public ASTWalker {
       verifyCheckedBase(E);
     }
 
+    void verifyChecked(BorrowExpr *E) {
+      PrettyStackTraceExpr debugStack(Ctx, "verifying BorrowExpr", E);
+
+      auto toType = E->getType();
+      auto fromType = E->getSubExpr()->getType();
+
+      if (!fromType->hasLValueType())
+        error("borrow source must be an l-value", E);
+
+      if (toType->hasLValueType())
+        error("borrow result must be an r-value", E);
+    }
+
     void verifyChecked(ABISafeConversionExpr *E) {
       PrettyStackTraceExpr debugStack(Ctx, "verify ABISafeConversionExpr", E);
 

lib/AST/FeatureSet.cpp

@@ -204,6 +204,7 @@ static bool usesFeatureInlineArrayTypeSugar(Decl *D) {
 }
 
 UNINTERESTING_FEATURE(StaticExclusiveOnly)
+UNINTERESTING_FEATURE(ManualOwnership)
 UNINTERESTING_FEATURE(ExtractConstantsFromMembers)
 UNINTERESTING_FEATURE(GroupActorErrors)
 UNINTERESTING_FEATURE(SameElementRequirements)

lib/ASTGen/Sources/ASTGen/DeclAttrs.swift

@@ -266,6 +266,7 @@ extension ASTGenVisitor {
         .noLocks,
         .noMetadata,
         .noObjCBridging,
+        .manualOwnership,
         .nonEphemeral,
         .nonEscapable,
         .nonObjC,

lib/SIL/IR/SILFunctionBuilder.cpp

@@ -202,6 +202,8 @@ void SILFunctionBuilder::addFunctionAttributes(
     F->setPerfConstraints(PerformanceConstraints::NoExistentials);
   } else if (Attrs.hasAttribute<NoObjCBridgingAttr>()) {
     F->setPerfConstraints(PerformanceConstraints::NoObjCBridging);
+  } else if (Attrs.hasAttribute<ManualOwnershipAttr>()) {
+    F->setPerfConstraints(PerformanceConstraints::ManualOwnership);
   }
 
   if (Attrs.hasAttribute<LexicalLifetimesAttr>()) {

lib/SIL/IR/SILPrinter.cpp

@@ -3635,6 +3635,7 @@ void SILFunction::print(SILPrintContext &PrintCtx) const {
     case PerformanceConstraints::NoRuntime:      OS << "[no_runtime] "; break;
     case PerformanceConstraints::NoExistentials: OS << "[no_existentials] "; break;
     case PerformanceConstraints::NoObjCBridging: OS << "[no_objc_bridging] "; break;
+    case PerformanceConstraints::ManualOwnership: OS << "[manual_ownership] "; break;
   }
 
   if (isPerformanceConstraint())

lib/SIL/Parser/ParseSIL.cpp

@@ -782,6 +782,8 @@ static bool parseDeclSILOptional(
       *perfConstraints = PerformanceConstraints::NoExistentials;
     else if (perfConstraints && SP.P.Tok.getText() == "no_objc_bridging")
       *perfConstraints = PerformanceConstraints::NoObjCBridging;
+    else if (perfConstraints && SP.P.Tok.getText() == "manual_ownership")
+      *perfConstraints = PerformanceConstraints::ManualOwnership;
     else if (isPerformanceConstraint && SP.P.Tok.getText() == "perf_constraint")
       *isPerformanceConstraint = true;
     else if (markedAsUsed && SP.P.Tok.getText() == "used")

lib/SIL/Utils/InstructionUtils.cpp

@@ -567,7 +567,6 @@ RuntimeEffect swift::getRuntimeEffect(SILInstruction *inst, SILType &impactType)
   case SILInstructionKind::CopyableToMoveOnlyWrapperValueInst:
   case SILInstructionKind::MoveOnlyWrapperToCopyableValueInst:
   case SILInstructionKind::UncheckedOwnershipConversionInst:
-  case SILInstructionKind::LoadInst:
   case SILInstructionKind::LoadBorrowInst:
   case SILInstructionKind::BeginBorrowInst:
   case SILInstructionKind::BorrowedFromInst:
@@ -642,6 +641,13 @@ RuntimeEffect swift::getRuntimeEffect(SILInstruction *inst, SILType &impactType)
   case SILInstructionKind::TypeValueInst:
   case SILInstructionKind::IgnoredUseInst:
     return RuntimeEffect::NoEffect;
+
+  case SILInstructionKind::LoadInst:
+    // FIXME: this needs to be merged in a separate PR as it can affect other performance constraints
+    if (cast<LoadInst>(inst)->getOwnershipQualifier() == LoadOwnershipQualifier::Copy)
+      return RuntimeEffect::RefCounting;
+    else
+      return RuntimeEffect::NoEffect;
 
   case SILInstructionKind::OpenExistentialMetatypeInst:
   case SILInstructionKind::OpenExistentialBoxInst:

lib/SIL/Utils/SILBridging.cpp

@@ -179,6 +179,7 @@ static_assert((int)BridgedFunction::PerformanceConstraints::NoLocks == (int)swif
 static_assert((int)BridgedFunction::PerformanceConstraints::NoRuntime == (int)swift::PerformanceConstraints::NoRuntime);
 static_assert((int)BridgedFunction::PerformanceConstraints::NoExistentials == (int)swift::PerformanceConstraints::NoExistentials);
 static_assert((int)BridgedFunction::PerformanceConstraints::NoObjCBridging == (int)swift::PerformanceConstraints::NoObjCBridging);
+static_assert((int)BridgedFunction::PerformanceConstraints::ManualOwnership == (int)swift::PerformanceConstraints::ManualOwnership);
 
 static_assert((int)BridgedFunction::InlineStrategy::InlineDefault == (int)swift::InlineDefault);
 static_assert((int)BridgedFunction::InlineStrategy::NoInline == (int)swift::NoInline);

lib/SILGen/SILGenDecl.cpp

@@ -1664,6 +1664,8 @@ void SILGenFunction::emitPatternBinding(PatternBindingDecl *PBD, unsigned idx,
     }
   }
 
+  // Form an evaluation scope for the initializing expression
+  FormalEvaluationScope writeback(*this);
   emitExprInto(initExpr, initialization.get());
 }
 

lib/SILGen/SILGenExpr.cpp

@@ -466,6 +466,7 @@ namespace {
 
     RValue visitStringLiteralExpr(StringLiteralExpr *E, SGFContext C);
     RValue visitLoadExpr(LoadExpr *E, SGFContext C);
+    RValue visitBorrowExpr(BorrowExpr *E, SGFContext C);
     RValue visitDerivedToBaseExpr(DerivedToBaseExpr *E, SGFContext C);
     RValue visitMetatypeConversionExpr(MetatypeConversionExpr *E,
                                        SGFContext C);
@@ -1108,6 +1109,20 @@ RValue RValueEmitter::visitLoadExpr(LoadExpr *E, SGFContext C) {
                               C.withFollowingSideEffects());
 }
 
+RValue RValueEmitter::visitBorrowExpr(BorrowExpr *E, SGFContext C_Ignored) {
+  // FIXME: how about address-only types? Maybe BorrowedAddressRead?
+
+  // NOTE: You should NOT add an evaluation scope here!
+  // The callers of this visitor should have established a scope already that
+  // encompasses the use of the borrowed RValue that we return.
+  assert(SGF.isInFormalEvaluationScope() && "emit borrow_expr without scope?");
+
+  LValue lv = SGF.emitLValue(E->getSubExpr(), SGFAccessKind::BorrowedObjectRead);
+  auto substFormalType = lv.getSubstFormalType();
+  ManagedValue mv = SGF.emitBorrowedLValue(E, std::move(lv));
+  return RValue(SGF, E, substFormalType, mv);
+}
+
 SILValue SILGenFunction::emitTemporaryAllocation(SILLocation loc, SILType ty,
                                                  HasDynamicLifetime_t dynamic,
                                                  IsLexical_t isLexical,
@@ -7222,7 +7237,10 @@ RValue RValueEmitter::visitCopyExpr(CopyExpr *E, SGFContext C) {
   }
 
   if (subType.isLoadable(SGF.F)) {
-    ManagedValue mv = SGF.emitRValue(subExpr).getAsSingleValue(SGF, subExpr);
+    // FIXME: this is a general improvement that can land in a separate PR
+    ManagedValue mv =
+      SGF.emitRValue(subExpr, SGFContext::AllowImmediatePlusZero)
+         .getAsSingleValue(SGF, subExpr);
     if (mv.getType().isTrivial(SGF.F))
       return RValue(SGF, {mv}, subType.getASTType());
     {

lib/SILOptimizer/Mandatory/PerformanceDiagnostics.cpp

@@ -293,6 +293,11 @@ bool PerformanceDiagnostics::visitCallee(SILInstruction *callInst,
                                          CalleeList callees,
                                          PerformanceConstraints perfConstr,
                                          LocWithParent *parentLoc) {
+  // Manual Ownership is not checked recursively within callees of the function,
+  // it's a local, non-viral performance annotation.
+  if (perfConstr == PerformanceConstraints::ManualOwnership)
+    return false;
+
   LocWithParent asLoc(callInst->getLoc().getSourceLoc(), parentLoc);
   LocWithParent *loc = &asLoc;
   if (parentLoc && asLoc.loc == callInst->getFunction()->getLocation().getSourceLoc())
@@ -370,6 +375,66 @@ bool PerformanceDiagnostics::visitInst(SILInstruction *inst,
   RuntimeEffect impact = getRuntimeEffect(inst, impactType);
   LocWithParent loc(inst->getLoc().getSourceLoc(), parentLoc);
 
+  if (perfConstr == PerformanceConstraints::ManualOwnership) {
+    if (impact == RuntimeEffect::RefCounting) {
+      bool shouldDiagnose = false;
+      switch (inst->getKind()) {
+      case SILInstructionKind::ExplicitCopyAddrInst:
+      case SILInstructionKind::ExplicitCopyValueInst:
+        break;
+      case SILInstructionKind::LoadInst: {
+        // FIXME: we don't have an `explicit_load` and transparent functions can
+        //   end up bringing in a `load [copy]`. A better approach is needed to
+        //   handle transparent functions, in general, as rewriting them during
+        //   inlining of transparent functions is also not so great, as it
+        //   may hide a copy that semantically is there, but we happened to
+        //   tuck away in a transparent synthesized function, like those
+        //   synthesized getters!
+        //
+        // For now look to see if the load copy's only non-destroying users are
+        // explicit_copy's, as that would indicate the user has acknowledged it:
+        //
+        //  %y = load [copy] %x
+        //  %z = explicit_copy_value %y
+        //  destroy_value %y
+        //
+        // In all other cases, it's safer to diagnose.
+        //
+        auto load = cast<LoadInst>(inst);
+        if (load->getOwnershipQualifier() != LoadOwnershipQualifier::Copy)
+          break;
+
+        for (auto *use : load->getUsers()) {
+          if (!isa<ExplicitCopyAddrInst, ExplicitCopyValueInst,
+                   DestroyAddrInst, DestroyValueInst>(use)) {
+            shouldDiagnose = true;
+            break;
+          }
+        }
+        break;
+      }
+      default:
+        shouldDiagnose = true;
+        break;
+      }
+      if (shouldDiagnose) {
+        diagnose(loc, diag::manualownership_copy);
+        llvm::dbgs() << "function " << inst->getFunction()->getName();
+        llvm::dbgs() << "\n has unexpected copying instruction: ";
+        inst->dump();
+        return false; // Don't bail-out early; diagnose more issues in the func.
+      }
+    } else if (impact == RuntimeEffect::ExclusivityChecking) {
+      // TODO: diagnose only the nested exclusivity; perhaps as a warning
+      //   since we don't yet have reference bindings?
+
+      // diagnose(loc, diag::performance_arc);
+      // inst->dump();
+      // return true;
+    }
+    return false;
+  }
+
   if (perfConstr == PerformanceConstraints::NoExistentials &&
       ((impact & RuntimeEffect::Existential) ||
        (impact & RuntimeEffect::ExistentialClassBound))) {

lib/Sema/CSApply.cpp

@@ -7125,6 +7125,15 @@ Expr *ExprRewriter::coerceExistential(Expr *expr, Type toType,
 }
 
 Expr *ConstraintSystem::addImplicitLoadExpr(Expr *expr) {
+  /// Avoid copying if we're in a manual ownership context; borrow instead.
+  if (auto *decl = DC->getAsDecl()) {
+    if (decl->getAttrs().hasAttribute<ManualOwnershipAttr>()) {
+      return TypeChecker::addImplicitBorrowExpr(
+        getASTContext(), expr, [this](Expr *expr) { return getType(expr); },
+        [this](Expr *expr, Type type) { setType(expr, type); });
+    }
+  }
+
   return TypeChecker::addImplicitLoadExpr(
       getASTContext(), expr, [this](Expr *expr) { return getType(expr); },
       [this](Expr *expr, Type type) { setType(expr, type); });