Skip to content

activator: assign multicast publisher IPs from global pool in GlobalConfig #2806

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
martinsander00 wants to merge 1 commit into
base: main
Choose a base branch
from
+996 −97
Open

activator: assign multicast publisher IPs from global pool in GlobalConfig #2806

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ All notable changes to this project will be documented in this file.

## Unreleased

- Activator
- Assign multicast publisher IPs from global pool in serviceability GlobalConfig instead of per-device blocks
- CLI
- Remove log noise on resolve route
- Onchain programs
Expand Down
1 change: 1 addition & 0 deletions activator/src/activator.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -400,6 +400,7 @@ mod tests {
device_tunnel_block: "1.0.0.0/24".parse().unwrap(),
user_tunnel_block: "1.0.1.0/24".parse().unwrap(),
multicastgroup_block: "239.239.239.0/24".parse().unwrap(),
multicast_publisher_block: "147.51.126.0/23".parse().unwrap(),
next_bgp_community: 65535,
};
mock_client
Expand Down
173 changes: 148 additions & 25 deletions activator/src/process/user.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,13 @@ use doublezero_program_common::types::NetworkV4;
use doublezero_sdk::{
commands::{
device::get::GetDeviceCommand,
resource::{allocate::AllocateResourceCommand, deallocate::DeallocateResourceCommand},
user::{
activate::ActivateUserCommand, ban::BanUserCommand,
closeaccount::CloseAccountUserCommand, reject::RejectUserCommand,
},
},
DoubleZeroClient, Exchange, Location, User, UserStatus, UserType,
DoubleZeroClient, Exchange, IdOrIp, Location, ResourceType, User, UserStatus, UserType,
};
use doublezero_serviceability::error::DoubleZeroError;
use log::{info, warn};
Expand All @@ -28,6 +29,7 @@ pub fn process_user_event(
pubkey: &Pubkey,
devices: &mut DeviceMap,
user_tunnel_ips: &mut IPBlockAllocator,
publisher_dz_ips: &mut Option<IPBlockAllocator>,
link_ids: &mut IDAllocator,
user: &User,
locations: &HashMap<Pubkey, Location>,
Expand Down Expand Up @@ -105,50 +107,133 @@ pub fn process_user_event(
UserType::Multicast => !user.publishers.is_empty(),
};

let dz_ip = if need_dz_ip {
match device_state.get_next_dz_ip() {
Some(ip) => ip,
None => {
let res =
reject_user(client, pubkey, "Error: No available dz_ip to allocate");
// Determine allocation strategy for dz_ip:
// - Multicast publishers: use publisher_dz_ips pool if available, otherwise onchain
// - Other types: respect use_onchain_allocation flag
let is_publisher = user.user_type == UserType::Multicast && !user.publishers.is_empty();
let use_onchain_dz_ip = if is_publisher {
use_onchain_allocation || publisher_dz_ips.is_none()
} else {
use_onchain_allocation
};

match res {
Ok(signature) => {
write!(
&mut log_msg,
" Reject(No available dz_ip to allocate) Rejected {signature}"
)
.unwrap();
let dz_ip = if need_dz_ip && !use_onchain_dz_ip {
// Offchain allocation
if is_publisher {
// Publishers: allocate from global publisher pool
if let Some(ref mut publisher_ips) = publisher_dz_ips {
match publisher_ips.next_available_block(1, 1).map(|net| net.ip()) {
Some(ip) => {
// Sync off-chain allocation to on-chain bitmap
// This ensures on-chain allocator knows about off-chain allocations
if let Ok(ip_net) = NetworkV4::new(ip, 32) {
let sync_result = AllocateResourceCommand {
resource_type: ResourceType::MulticastPublisherBlock,
requested: Some(IdOrIp::Ip(ip_net)),
}
.execute(client);

if let Err(e) = sync_result {
warn!(
"Failed to sync off-chain publisher IP {} to on-chain: {}",
ip, e
);
// Continue anyway - off-chain allocation is still valid
}
}
ip
}
Err(e) => {
write!(
&mut log_msg,
" Reject(No available dz_ip to allocate) Error: {e}"
)
.unwrap();
None => {
let res = reject_user(
client,
pubkey,
"Error: No available publisher dz_ip to allocate",
);

match res {
Ok(signature) => {
write!(
&mut log_msg,
" Reject(No available publisher dz_ip) Rejected {signature}"
)
.unwrap();
}
Err(e) => {
write!(
&mut log_msg,
" Reject(No available publisher dz_ip) Error: {e}"
)
.unwrap();
}
}
info!("{log_msg}");
return;
}
}
info!("{log_msg}");
return;
} else {
// Should never happen due to use_onchain_dz_ip check above
Ipv4Addr::UNSPECIFIED
}
} else {
// IBRL/EdgeFiltering: allocate from device state
match device_state.get_next_dz_ip() {
Some(ip) => ip,
None => {
let res = reject_user(
client,
pubkey,
"Error: No available dz_ip to allocate",
);

match res {
Ok(signature) => {
write!(
&mut log_msg,
" Reject(No available dz_ip to allocate) Rejected {signature}"
)
.unwrap();
}
Err(e) => {
write!(
&mut log_msg,
" Reject(No available dz_ip to allocate) Error: {e}"
)
.unwrap();
}
}
info!("{log_msg}");
return;
}
}
}
} else if need_dz_ip {
// Onchain allocation: pass UNSPECIFIED so smart contract allocates
Ipv4Addr::UNSPECIFIED
} else {
user.client_ip
};

write!(&mut log_msg, " tunnel_id: {} dz_ip: {} ", tunnel_id, &dz_ip).unwrap();

// Activate the user
// Force onchain allocation for multicast publishers if no local publisher pool
let use_onchain_for_activation =
use_onchain_allocation || (is_publisher && publisher_dz_ips.is_none());

let res = ActivateUserCommand {
user_pubkey: *pubkey,
tunnel_id: if use_onchain_allocation { 0 } else { tunnel_id },
tunnel_net: if use_onchain_allocation {
tunnel_id: if use_onchain_for_activation {
0
} else {
tunnel_id
},
tunnel_net: if use_onchain_for_activation {
NetworkV4::default()
} else {
tunnel_net.into()
},
dz_ip,
use_onchain_allocation,
use_onchain_allocation: use_onchain_for_activation,
}
.execute(client);

Expand Down Expand Up @@ -312,6 +397,37 @@ pub fn process_user_event(
if user.tunnel_net != NetworkV4::default() {
user_tunnel_ips.unassign_block(user.tunnel_net.into());
}
// Deallocate publisher dz_ip from global pool
if user.user_type == UserType::Multicast
&& !user.publishers.is_empty()
&& user.dz_ip != Ipv4Addr::UNSPECIFIED
&& user.dz_ip != user.client_ip
{
if let Some(ref mut publisher_ips) = publisher_dz_ips {
if let Ok(dz_ip_net) = NetworkV4::new(user.dz_ip, 32) {
publisher_ips.unassign_block(dz_ip_net.into());
info!(
"Deallocated publisher dz_ip {} from global pool",
user.dz_ip
);

// Sync deallocation to on-chain bitmap
let sync_result = DeallocateResourceCommand {
resource_type:
ResourceType::MulticastPublisherBlock,
value: IdOrIp::Ip(dz_ip_net),
}
.execute(client);

if let Err(e) = sync_result {
warn!(
"Failed to sync off-chain publisher IP {} deallocation to on-chain: {}",
user.dz_ip, e
);
}
}
}
}
}
if user.dz_ip != Ipv4Addr::UNSPECIFIED {
device_state.release(user.dz_ip, user.tunnel_id).unwrap();
Expand Down Expand Up @@ -575,6 +691,7 @@ mod tests {
&user_pubkey,
&mut devices,
&mut user_tunnel_ips,
&mut None, // publisher_dz_ips
&mut link_ids,
&user,
&locations,
Expand Down Expand Up @@ -771,6 +888,7 @@ mod tests {
&user_pubkey,
&mut devices,
&mut user_tunnel_ips,
&mut None, // publisher_dz_ips
&mut link_ids,
&user,
&locations,
Expand Down Expand Up @@ -875,6 +993,7 @@ mod tests {
&user_pubkey,
&mut devices,
&mut user_tunnel_ips,
&mut None, // publisher_dz_ips
&mut link_ids,
&user,
&locations,
Expand Down Expand Up @@ -983,6 +1102,7 @@ mod tests {
&user_pubkey,
&mut devices,
&mut user_tunnel_ips,
&mut None, // publisher_dz_ips
&mut link_ids,
&user,
&locations,
Expand Down Expand Up @@ -1088,6 +1208,7 @@ mod tests {
&user_pubkey,
&mut devices,
&mut user_tunnel_ips,
&mut None, // publisher_dz_ips
&mut link_ids,
&user,
&locations,
Expand Down Expand Up @@ -1194,6 +1315,7 @@ mod tests {
&user_pubkey,
&mut devices,
&mut user_tunnel_ips,
&mut None, // publisher_dz_ips
&mut link_ids,
&user,
&locations,
Expand Down Expand Up @@ -1248,6 +1370,7 @@ mod tests {
predicate::eq(DoubleZeroInstruction::CloseAccountUser(
UserCloseAccountArgs {
dz_prefix_count: 0, // legacy path
multicast_publisher_count: 0,
},
)),
predicate::always(),
Expand Down
26 changes: 25 additions & 1 deletion activator/src/processor.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ use crate::{
states::devicestate::DeviceState,
};
use backon::{BlockingRetryable, ExponentialBuilder};
use doublezero_program_common::types::NetworkV4;
use doublezero_sdk::{
commands::{
device::list::ListDeviceCommand, exchange::list::ListExchangeCommand,
Expand All @@ -18,7 +19,7 @@ use doublezero_sdk::{
},
doublezeroclient::DoubleZeroClient,
AccountData, DeviceStatus, Exchange, GetGlobalConfigCommand, InterfaceType, LinkStatus,
Location, MulticastGroup, UserStatus,
Location, MulticastGroup, UserStatus, UserType,
};
use log::{debug, error, info, warn};
use solana_sdk::pubkey::Pubkey;
Expand All @@ -42,6 +43,7 @@ pub struct Processor<T: DoubleZeroClient> {
link_ips: IPBlockAllocator,
multicastgroup_tunnel_ips: IPBlockAllocator,
user_tunnel_ips: IPBlockAllocator,
publisher_dz_ips: Option<IPBlockAllocator>,
devices: DeviceMap,
locations: LocationMap,
exchanges: ExchangeMap,
Expand Down Expand Up @@ -79,6 +81,9 @@ impl<T: DoubleZeroClient> Processor<T> {
let mut link_ips = IPBlockAllocator::new(config.device_tunnel_block.into());
let mut segment_routing_ids = IDAllocator::new(1, vec![]);
let mut user_tunnel_ips = IPBlockAllocator::new(config.user_tunnel_block.into());
let mut publisher_dz_ips = Some(IPBlockAllocator::new(
config.multicast_publisher_block.into(),
));

for (_, link) in links
.iter()
Expand Down Expand Up @@ -122,6 +127,23 @@ impl<T: DoubleZeroClient> Processor<T> {
)
})?;
user_tunnel_ips.assign_block(user.tunnel_net.into());

// Mark publisher IPs as allocated in the publisher pool
if user.user_type == UserType::Multicast
&& !user.publishers.is_empty()
&& user.dz_ip != std::net::Ipv4Addr::UNSPECIFIED
&& user.dz_ip != user.client_ip
{
if let Some(ref mut publisher_ips) = publisher_dz_ips {
if let Ok(dz_ip_net) = NetworkV4::new(user.dz_ip, 32) {
publisher_ips.assign_block(dz_ip_net.into());
info!(
"Marked publisher dz_ip {} as allocated (loaded from existing user)",
user.dz_ip
);
}
}
}
}
Ok::<(), eyre::Error>(())
})?;
Expand All @@ -141,6 +163,7 @@ impl<T: DoubleZeroClient> Processor<T> {
segment_routing_ids,
multicastgroup_tunnel_ips: IPBlockAllocator::new(config.multicastgroup_block.into()),
user_tunnel_ips,
publisher_dz_ips,
devices: device_map,
locations,
exchanges,
Expand Down Expand Up @@ -190,6 +213,7 @@ impl<T: DoubleZeroClient> Processor<T> {
pubkey,
&mut self.devices,
&mut self.user_tunnel_ips,
&mut self.publisher_dz_ips,
&mut self.link_ids,
user,
&self.locations,
Expand Down
1 change: 1 addition & 0 deletions activator/src/tests.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ pub mod utils {
device_tunnel_block: "1.0.0.0/24".parse().unwrap(),
user_tunnel_block: "2.0.0.0/24".parse().unwrap(),
multicastgroup_block: "224.0.0.0/24".parse().unwrap(),
multicast_publisher_block: "147.51.126.0/23".parse().unwrap(),
next_bgp_community: 0,
};

Expand Down
1 change: 1 addition & 0 deletions client/doublezero/src/command/connect.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -822,6 +822,7 @@ mod tests {
device_tunnel_block: "10.0.0.0/24".parse().unwrap(),
user_tunnel_block: "10.0.1.0/24".parse().unwrap(),
multicastgroup_block: "239.0.0.0/24".parse().unwrap(),
multicast_publisher_block: "147.51.126.0/23".parse().unwrap(),
next_bgp_community: 10000,
},
client: create_test_client(),
Expand Down
4 changes: 2 additions & 2 deletions e2e/internal/devnet/smartcontract_init.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,8 +91,8 @@ func (dn *Devnet) InitSmartContract(ctx context.Context) error {

# Populate global configuration onchain.
echo "==> Populating global configuration onchain"
echo doublezero global-config set --local-asn 65000 --remote-asn 65342 --device-tunnel-block ` + dn.Spec.DeviceTunnelNet + ` --user-tunnel-block 169.254.0.0/16 --multicastgroup-block 233.84.178.0/24
doublezero global-config set --local-asn 65000 --remote-asn 65342 --device-tunnel-block ` + dn.Spec.DeviceTunnelNet + ` --user-tunnel-block 169.254.0.0/16 --multicastgroup-block 233.84.178.0/24
echo doublezero global-config set --local-asn 65000 --remote-asn 65342 --device-tunnel-block ` + dn.Spec.DeviceTunnelNet + ` --user-tunnel-block 169.254.0.0/16 --multicastgroup-block 233.84.178.0/24 --multicast-publisher-block 147.51.126.0/23
doublezero global-config set --local-asn 65000 --remote-asn 65342 --device-tunnel-block ` + dn.Spec.DeviceTunnelNet + ` --user-tunnel-block 169.254.0.0/16 --multicastgroup-block 233.84.178.0/24 --multicast-publisher-block 147.51.126.0/23
echo "--> Global configuration onchain:"
doublezero global-config get
echo
Expand Down
Loading
Loading