Skip to content

Commit 1bd96dd

Browse files
author
pixcc
committed
Add local registration in shared hive KIKIMR-20108
Add local registration in shared hive KIKIMR-20108
1 parent 591b2e7 commit 1bd96dd

File tree

9 files changed

+130
-17
lines changed

9 files changed

+130
-17
lines changed

‎ydb/core/mind/hive/hive_ut.cpp

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5742,6 +5742,85 @@ Y_UNIT_TEST_SUITE(THiveTest) {
57425742
WaitForTabletIsUp(runtime, tablet, 1);
57435743
}
57445744
}
5745+
5746+
Y_UNIT_TEST(TestLocalRegistrationInSharedHive) {
5747+
TTestBasicRuntime runtime(2, false);
5748+
Setup(runtime, true);
5749+
5750+
const ui64 hiveTablet = MakeDefaultHiveID(0);
5751+
const ui64 testerTablet = MakeDefaultHiveID(1);
5752+
CreateTestBootstrapper(runtime, CreateTestTabletInfo(hiveTablet, TTabletTypes::Hive), &CreateDefaultHive);
5753+
CreateTestBootstrapper(runtime, CreateTestTabletInfo(TTestTxConfig::SchemeShard, TTabletTypes::SchemeShard), &CreateFlatTxSchemeShard);
5754+
MakeSureTabletIsUp(runtime, hiveTablet, 0); // root hive good
5755+
MakeSureTabletIsUp(runtime, TTestTxConfig::SchemeShard, 0); // root ss good
5756+
5757+
TActorId sender = runtime.AllocateEdgeActor(0);
5758+
InitSchemeRoot(runtime, sender);
5759+
5760+
// Create subdomain
5761+
ui32 txId = 100;
5762+
TSubDomainKey subdomainKey;
5763+
do {
5764+
auto modifyScheme = MakeHolder<NSchemeShard::TEvSchemeShard::TEvModifySchemeTransaction>();
5765+
modifyScheme->Record.SetTxId(++txId);
5766+
auto* transaction = modifyScheme->Record.AddTransaction();
5767+
transaction->SetWorkingDir("/dc-1");
5768+
transaction->SetOperationType(NKikimrSchemeOp::ESchemeOpCreateExtSubDomain);
5769+
auto* subdomain = transaction->MutableSubDomain();
5770+
subdomain->SetName("tenant1");
5771+
runtime.SendToPipe(TTestTxConfig::SchemeShard, sender, modifyScheme.Release());
5772+
TAutoPtr<IEventHandle> handle;
5773+
auto reply = runtime.GrabEdgeEventRethrow<NSchemeShard::TEvSchemeShard::TEvModifySchemeTransactionResult>(handle, TDuration::MilliSeconds(100));
5774+
if (reply) {
5775+
subdomainKey = TSubDomainKey(reply->Record.GetSchemeshardId(), reply->Record.GetPathId());
5776+
UNIT_ASSERT_VALUES_EQUAL(reply->Record.GetStatus(), NKikimrScheme::EStatus::StatusAccepted);
5777+
break;
5778+
}
5779+
} while (true);
5780+
5781+
// Create shared hive
5782+
THolder<TEvHive::TEvCreateTablet> createSharedHive = MakeHolder<TEvHive::TEvCreateTablet>(testerTablet, 0, TTabletTypes::Hive, BINDED_CHANNELS);
5783+
createSharedHive->Record.AddAllowedDomains();
5784+
createSharedHive->Record.MutableAllowedDomains(0)->SetSchemeShard(TTestTxConfig::SchemeShard);
5785+
createSharedHive->Record.MutableAllowedDomains(0)->SetPathId(1);
5786+
ui64 sharedHiveTablet = SendCreateTestTablet(runtime, hiveTablet, testerTablet, std::move(createSharedHive), 0, false);
5787+
MakeSureTabletIsUp(runtime, sharedHiveTablet, 0); // shared hive good
5788+
5789+
// Setup resolving shared hive for subdomain
5790+
runtime.SetObserverFunc([&](TAutoPtr<IEventHandle>& event) {
5791+
if (event->GetTypeRewrite() == NSchemeShard::TEvSchemeShard::EvDescribeSchemeResult) {
5792+
auto* record = event->Get<NSchemeShard::TEvSchemeShard::TEvDescribeSchemeResult>()->MutableRecord();
5793+
TSubDomainKey resolvingSubdomainKey(record->GetPathOwnerId(), record->GetPathId());
5794+
if (resolvingSubdomainKey == subdomainKey) {
5795+
record->MutablePathDescription()->MutableDomainDescription()->SetSharedHive(sharedHiveTablet);
5796+
}
5797+
}
5798+
return TTestActorRuntime::EEventAction::PROCESS;
5799+
});
5800+
5801+
// Start local for subdomain
5802+
SendKillLocal(runtime, 1);
5803+
CreateLocalForTenant(runtime, 1, "/dc-1/tenant1");
5804+
5805+
bool seenLocalRegistrationInSharedHive = false;
5806+
TTestActorRuntime::TEventObserver prevObserverFunc;
5807+
prevObserverFunc = runtime.SetObserverFunc([&](TAutoPtr<IEventHandle>& event) {
5808+
if (event->GetTypeRewrite() == TEvLocal::EvRegisterNode) {
5809+
const auto& record = event->Get<TEvLocal::TEvRegisterNode>()->Record;
5810+
if (record.GetHiveId() == sharedHiveTablet
5811+
&& !record.GetServicedDomains().empty()
5812+
&& TSubDomainKey(record.GetServicedDomains().Get(0)) == subdomainKey) {
5813+
seenLocalRegistrationInSharedHive = true;
5814+
}
5815+
}
5816+
return prevObserverFunc(event);
5817+
});
5818+
5819+
TDispatchOptions options;
5820+
options.FinalEvents.emplace_back(TEvLocal::EvRegisterNode, 2);
5821+
runtime.DispatchEvents(options);
5822+
UNIT_ASSERT(seenLocalRegistrationInSharedHive);
5823+
}
57455824
}
57465825

57475826
}

‎ydb/core/mind/local.cpp

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1224,25 +1224,29 @@ class TDomainLocal : public TActorBootstrapped<TDomainLocal> {
12241224
return;
12251225
}
12261226
Y_ABORT_UNLESS(rec.GetPathDescription().HasDomainDescription());
1227-
Y_ABORT_UNLESS(rec.GetPathDescription().GetDomainDescription().GetDomainKey().GetSchemeShard() == SchemeRoot);
1227+
const auto &domainDesc = rec.GetPathDescription().GetDomainDescription();
1228+
Y_ABORT_UNLESS(domainDesc.GetDomainKey().GetSchemeShard() == SchemeRoot);
12281229

12291230
TVector<TTabletId> hiveIds(HiveIds);
1230-
TString path = rec.GetPath();
1231-
1232-
TTabletId hiveId = rec.GetPathDescription().GetDomainDescription().GetProcessingParams().GetHive();
1231+
TTabletId hiveId = domainDesc.GetProcessingParams().GetHive();
12331232
if (hiveId) {
12341233
hiveIds.emplace_back(hiveId);
12351234
}
1235+
TTabletId sharedHiveId = domainDesc.GetSharedHive();
1236+
if (sharedHiveId) {
1237+
hiveIds.emplace_back(sharedHiveId);
1238+
}
12361239
RegisterAsSubDomain(rec, task, hiveIds, ctx);
12371240

1241+
const TString &path = rec.GetPath();
12381242
auto itTenant = RunningTenants.find(path);
12391243
if (itTenant != RunningTenants.end()) {
12401244
TTenantInfo& tenant = itTenant->second;
12411245

12421246
tenant.HiveIds = hiveIds;
12431247

1244-
SendStatus(rec.GetPath(), task.Senders, ctx);
1245-
ResolveTasks.erase(rec.GetPath());
1248+
SendStatus(path, task.Senders, ctx);
1249+
ResolveTasks.erase(path);
12461250

12471251
// subscribe for schema updates
12481252
const auto& domains = *AppData()->DomainsInfo;

‎ydb/core/protos/subdomains.proto

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,7 @@ message TDomainDescription {
101101
optional NLoginProto.TSecurityState SecurityState = 20;
102102

103103
optional TAuditSettings AuditSettings = 21;
104+
optional fixed64 SharedHive = 22;
104105
}
105106

106107
message TSchemeQuotas {

‎ydb/core/tx/schemeshard/schemeshard_path_describer.cpp

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -712,6 +712,10 @@ void TPathDescriber::DescribeDomainRoot(TPathElement::TPtr pathEl) {
712712
if (const auto& auditSettings = subDomainInfo->GetAuditSettings()) {
713713
entry->MutableAuditSettings()->CopyFrom(*auditSettings);
714714
}
715+
716+
if (TTabletId sharedHive = subDomainInfo->GetSharedHive()) {
717+
entry->SetSharedHive(sharedHive.GetValue());
718+
}
715719
}
716720

717721
void TPathDescriber::DescribeDomainExtra(TPathElement::TPtr pathEl) {

‎ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -212,6 +212,19 @@ TCheckFunc StoragePoolsEqual(TSet<TString> poolNames) {
212212
};
213213
}
214214

215+
TCheckFunc SharedHive(ui64 sharedHiveId) {
216+
return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) {
217+
UNIT_ASSERT_C(IsGoodDomainStatus(record.GetStatus()), "Unexpected status: " << record.GetStatus());
218+
219+
const auto& domainDesc = record.GetPathDescription().GetDomainDescription();
220+
if (sharedHiveId) {
221+
UNIT_ASSERT_VALUES_EQUAL(domainDesc.GetSharedHive(), sharedHiveId);
222+
} else {
223+
UNIT_ASSERT(!domainDesc.HasSharedHive());
224+
}
225+
};
226+
}
227+
215228
TCheckFunc DomainCoordinators(TVector<ui64> coordinators) {
216229
return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) {
217230
UNIT_ASSERT_C(IsGoodDomainStatus(record.GetStatus()), "Unexpected status: " << record.GetStatus());

‎ydb/core/tx/schemeshard/ut_helpers/ls_checks.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -148,7 +148,8 @@ namespace NLs {
148148

149149
TCheckFunc KesusConfigIs(ui64 self_check_period_millis, ui64 session_grace_period_millis);
150150
TCheckFunc DatabaseQuotas(ui64 dataStreamShards);
151-
151+
TCheckFunc SharedHive(ui64 sharedHiveId);
152+
152153
template<class TCheck>
153154
void PerformAllChecks(const NKikimrScheme::TEvDescribeSchemeResult& result, TCheck&& check) {
154155
check(result);

‎ydb/core/tx/schemeshard/ut_helpers/test_env.cpp

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -746,13 +746,13 @@ void NSchemeShardUT_Private::TTestEnv::TestWaitNotification(NActors::TTestActorR
746746
TestWaitNotification(runtime, ids, schemeshardId);
747747
}
748748

749-
void NSchemeShardUT_Private::TTestEnv::TestWaitTabletDeletion(NActors::TTestActorRuntime &runtime, TSet<ui64> tabletIds) {
749+
void NSchemeShardUT_Private::TTestEnv::TestWaitTabletDeletion(NActors::TTestActorRuntime &runtime, TSet<ui64> tabletIds, ui64 hive) {
750750
TActorId sender = runtime.AllocateEdgeActor();
751751

752752
for (ui64 tabletId : tabletIds) {
753753
Cerr << "wait until " << tabletId << " is deleted" << Endl;
754754
auto ev = new TEvFakeHive::TEvSubscribeToTabletDeletion(tabletId);
755-
ForwardToTablet(runtime, TTestTxConfig::Hive, sender, ev);
755+
ForwardToTablet(runtime, hive, sender, ev);
756756
}
757757

758758
TAutoPtr<IEventHandle> handle;
@@ -768,8 +768,8 @@ void NSchemeShardUT_Private::TTestEnv::TestWaitTabletDeletion(NActors::TTestActo
768768
}
769769
}
770770

771-
void NSchemeShardUT_Private::TTestEnv::TestWaitTabletDeletion(NActors::TTestActorRuntime &runtime, ui64 tabletId) {
772-
TestWaitTabletDeletion(runtime, TSet<ui64>{tabletId});
771+
void NSchemeShardUT_Private::TTestEnv::TestWaitTabletDeletion(NActors::TTestActorRuntime &runtime, ui64 tabletId, ui64 hive) {
772+
TestWaitTabletDeletion(runtime, TSet<ui64>{tabletId}, hive);
773773
}
774774

775775
void NSchemeShardUT_Private::TTestEnv::TestWaitShardDeletion(NActors::TTestActorRuntime &runtime, ui64 schemeShard, TSet<TShardIdx> shardIds) {

‎ydb/core/tx/schemeshard/ut_helpers/test_env.h

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -98,12 +98,12 @@ namespace NSchemeShardUT_Private {
9898
void TestWaitNotification(TTestActorRuntime& runtime, ui64 txId, ui64 schemeshardId = TTestTxConfig::SchemeShard);
9999

100100
template <class TContainer>
101-
void TestWaitTabletDeletion(TTestActorRuntime& runtime, TContainer tabletIds) {
101+
void TestWaitTabletDeletion(TTestActorRuntime& runtime, const TContainer& tabletIds, ui64 hive = TTestTxConfig::Hive) {
102102
TSet<ui64> set(tabletIds.begin(), tabletIds.end());
103-
TestWaitTabletDeletion(runtime, std::move(set));
103+
TestWaitTabletDeletion(runtime, std::move(set), hive);
104104
}
105-
void TestWaitTabletDeletion(TTestActorRuntime& runtime, TSet<ui64> tabletIds);
106-
void TestWaitTabletDeletion(TTestActorRuntime& runtime, ui64 tabletId);
105+
void TestWaitTabletDeletion(TTestActorRuntime& runtime, TSet<ui64> tabletIds, ui64 hive = TTestTxConfig::Hive);
106+
void TestWaitTabletDeletion(TTestActorRuntime& runtime, ui64 tabletId, ui64 hive = TTestTxConfig::Hive);
107107

108108
void TestWaitShardDeletion(TTestActorRuntime& runtime, TSet<ui64> localIds);
109109
void TestWaitShardDeletion(TTestActorRuntime& runtime, ui64 schemeShard, TSet<ui64> localIds);

‎ydb/core/tx/schemeshard/ut_serverless/ut_serverless.cpp

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,10 +34,19 @@ Y_UNIT_TEST_SUITE(TSchemeShardServerLess) {
3434
"Mediators: 1 "
3535
"TimeCastBucketsPerMediator: 2 "
3636
"ExternalSchemeShard: true "
37-
"ExternalHive: false " // ExternalHive is impossible in that environment yet
37+
"ExternalHive: true "
3838
"Name: \"SharedDB\"");
3939
env.TestWaitNotification(runtime, txId);
4040

41+
ui64 sharedHive = 0;
42+
TestDescribeResult(DescribePath(runtime, "/MyRoot/SharedDB"),
43+
{NLs::PathExist,
44+
NLs::IsExternalSubDomain("SharedDB"),
45+
NLs::ExtractDomainHive(&sharedHive)});
46+
UNIT_ASSERT(sharedHive != 0
47+
&& sharedHive != (ui64)-1
48+
&& sharedHive != TTestTxConfig::Hive);
49+
4150
TString createData = TStringBuilder()
4251
<< "ResourcesDomainKey { SchemeShard: " << TTestTxConfig::SchemeShard << " PathId: " << 2 << " } "
4352
<< "Name: \"ServerLess0\"";
@@ -63,6 +72,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardServerLess) {
6372
TestDescribeResult(DescribePath(runtime, "/MyRoot/ServerLess0"),
6473
{NLs::PathExist,
6574
NLs::IsExternalSubDomain("ServerLess0"),
75+
NLs::SharedHive(sharedHive),
6676
NLs::ExtractTenantSchemeshard(&tenantSchemeShard)});
6777

6878
UNIT_ASSERT(tenantSchemeShard != 0
@@ -94,7 +104,8 @@ Y_UNIT_TEST_SUITE(TSchemeShardServerLess) {
94104
NLs::PathsInsideDomain(1),
95105
NLs::ShardsInsideDomain(0)});
96106

97-
env.TestWaitTabletDeletion(runtime, xrange(TTestTxConfig::FakeHiveTablets + 3, TTestTxConfig::FakeHiveTablets + 10));
107+
ui64 sharedHiveTablets = TTestTxConfig::FakeHiveTablets + NKikimr::TFakeHiveState::TABLETS_PER_CHILD_HIVE;
108+
env.TestWaitTabletDeletion(runtime, xrange(sharedHiveTablets, sharedHiveTablets + 4), sharedHive);
98109
}
99110

100111
Y_UNIT_TEST(StorageBilling) {

0 commit comments

Comments
 (0)