From 0aa10995e20e1369f57b6cdce0d83b7d17951655 Mon Sep 17 00:00:00 2001 From: Eric Astor Date: Mon, 29 Dec 2025 14:35:02 -0800 Subject: [PATCH] [codegen 1.5] Update idle signal generation to account for active I/O handshakes. The idle signal now goes low not only when data is moving through the pipeline stages but also when input or output flops are actively completing a ready/valid handshake. This includes cases where function inputs are flopped or when flopped channels are completing a transaction. This lets us correctly enable some idle-signal block conversion tests after updating their expectations. PiperOrigin-RevId: 850162851 --- xls/codegen_v_1_5/BUILD | 76 ++- .../block_conversion_pass_pipeline.cc | 24 +- .../block_conversion_pass_pipeline.h | 5 +- .../block_conversion_pass_pipeline_test.cc | 638 ++++++++++++++---- .../block_conversion_wrapper_pass.cc | 32 + .../block_conversion_wrapper_pass.h | 74 ++ .../block_conversion_wrapper_pass_test.cc | 76 +++ .../channel_to_port_io_lowering_pass.cc | 229 +++++-- .../channel_to_port_io_lowering_pass_test.cc | 145 ++++ xls/codegen_v_1_5/convert_to_block.cc | 4 +- .../flow_control_insertion_pass.cc | 28 + .../function_io_lowering_pass.cc | 34 +- .../function_io_lowering_pass_test.cc | 31 +- xls/codegen_v_1_5/idle_insertion_pass.cc | 104 ++- xls/codegen_v_1_5/idle_insertion_pass.h | 13 + xls/codegen_v_1_5/register_cleanup_pass.cc | 188 ++++++ xls/codegen_v_1_5/register_cleanup_pass.h | 61 ++ .../state_to_register_io_lowering_pass.cc | 180 +++-- ..._io_lowering_test_SimpleProcWithFullBit.ir | 15 +- ...test_SimpleProcWithFullBitAndPredicates.ir | 16 +- ...est_SimpleProcWithMultipleStateElements.ir | 38 +- xls/ir/BUILD | 1 + xls/ir/function_base.cc | 6 + xls/ir/function_base.h | 4 +- xls/ir/name_uniquer.cc | 10 + xls/ir/name_uniquer.h | 12 + xls/ir/node_util.cc | 78 ++- xls/ir/node_util.h | 42 +- xls/ir/proc.cc | 4 + xls/scheduling/run_pipeline_schedule.cc | 4 +- 30 files changed, 1802 insertions(+), 370 deletions(-) create mode 100644 xls/codegen_v_1_5/block_conversion_wrapper_pass.cc create mode 100644 xls/codegen_v_1_5/block_conversion_wrapper_pass.h create mode 100644 xls/codegen_v_1_5/block_conversion_wrapper_pass_test.cc create mode 100644 xls/codegen_v_1_5/register_cleanup_pass.cc create mode 100644 xls/codegen_v_1_5/register_cleanup_pass.h diff --git a/xls/codegen_v_1_5/BUILD b/xls/codegen_v_1_5/BUILD index 2cceef3c91..9c5fe1b57c 100644 --- a/xls/codegen_v_1_5/BUILD +++ b/xls/codegen_v_1_5/BUILD @@ -41,15 +41,21 @@ cc_library( hdrs = ["block_conversion_pass_pipeline.h"], deps = [ ":block_conversion_pass", + ":block_conversion_wrapper_pass", ":block_finalization_pass", ":channel_to_port_io_lowering_pass", ":flow_control_insertion_pass", ":function_io_lowering_pass", ":idle_insertion_pass", ":pipeline_register_insertion_pass", + ":register_cleanup_pass", ":scheduled_block_conversion_pass", ":scheduling_pass", ":state_to_register_io_lowering_pass", + "//xls/passes:dataflow_simplification_pass", + "//xls/passes:dce_pass", + "//xls/passes:identity_removal_pass", + "//xls/passes:optimization_pass", ], ) @@ -93,12 +99,72 @@ cc_test( ], ) +cc_library( + name = "register_cleanup_pass", + srcs = ["register_cleanup_pass.cc"], + hdrs = ["register_cleanup_pass.h"], + deps = [ + ":block_conversion_pass", + "//xls/common/status:status_macros", + "//xls/data_structures:transitive_closure", + "//xls/ir", + "//xls/ir:register", + "//xls/ir:value", + "//xls/ir:value_utils", + "//xls/passes:node_dependency_analysis", + "//xls/passes:partial_info_query_engine", + "//xls/passes:pass_base", + "//xls/passes:query_engine", + "@com_google_absl//absl/algorithm:container", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/container:flat_hash_set", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/types:span", + ], +) + +cc_library( + name = "block_conversion_wrapper_pass", + srcs = ["block_conversion_wrapper_pass.cc"], + hdrs = ["block_conversion_wrapper_pass.h"], + deps = [ + ":block_conversion_pass", + "//xls/ir", + "//xls/passes:optimization_pass", + "//xls/passes:pass_base", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings:str_format", + "@com_google_absl//absl/types:span", + ], +) + +cc_test( + name = "block_conversion_wrapper_pass_test", + srcs = ["block_conversion_wrapper_pass_test.cc"], + deps = [ + ":block_conversion_pass", + ":block_conversion_wrapper_pass", + "//xls/common:xls_gunit_main", + "//xls/common/status:matchers", + "//xls/ir", + "//xls/ir:function_builder", + "//xls/ir:ir_matcher", + "//xls/ir:ir_test_base", + "//xls/passes:dce_pass", + "//xls/passes:optimization_pass", + "//xls/passes:pass_base", + "@com_google_absl//absl/status:statusor", + "@googletest//:gtest", + ], +) + cc_library( name = "channel_to_port_io_lowering_pass", srcs = ["channel_to_port_io_lowering_pass.cc"], hdrs = ["channel_to_port_io_lowering_pass.h"], deps = [ ":block_conversion_pass", + "//xls/codegen:codegen_options", "//xls/codegen:conversion_utils", "//xls/codegen:ram_configuration", "//xls/common:casts", @@ -170,6 +236,7 @@ cc_library( "//xls/common/status:status_macros", "//xls/estimators/delay_model:delay_estimator", "//xls/ir", + "//xls/passes:optimization_pass", "//xls/passes:pass_base", "//xls/scheduling:pipeline_schedule_cc_proto", "//xls/scheduling:scheduling_options", @@ -254,9 +321,9 @@ cc_library( "//xls/common/status:ret_check", "//xls/common/status:status_macros", "//xls/ir", + "//xls/ir:node_util", "//xls/ir:op", "//xls/ir:register", - "//xls/ir:value", "//xls/ir:value_utils", "//xls/passes:pass_base", "@com_google_absl//absl/algorithm:container", @@ -312,15 +379,21 @@ cc_library( hdrs = ["idle_insertion_pass.h"], deps = [ ":block_conversion_pass", + "//xls/codegen:codegen_options", "//xls/common:casts", "//xls/common/status:ret_check", "//xls/common/status:status_macros", "//xls/ir", + "//xls/ir:bits", + "//xls/ir:channel", "//xls/ir:node_util", "//xls/ir:op", "//xls/ir:source_location", + "//xls/ir:value", "//xls/passes:pass_base", + "@com_google_absl//absl/log:check", "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:str_format", "@com_google_absl//absl/types:span", ], @@ -563,7 +636,6 @@ cc_test( "//xls/common/status:matchers", "//xls/common/status:ret_check", "//xls/common/status:status_macros", - "//xls/estimators/delay_model:delay_estimator", "//xls/interpreter:block_evaluator", "//xls/interpreter:ir_interpreter", "//xls/ir", diff --git a/xls/codegen_v_1_5/block_conversion_pass_pipeline.cc b/xls/codegen_v_1_5/block_conversion_pass_pipeline.cc index ea3f42c842..a89cc1d390 100644 --- a/xls/codegen_v_1_5/block_conversion_pass_pipeline.cc +++ b/xls/codegen_v_1_5/block_conversion_pass_pipeline.cc @@ -20,20 +20,26 @@ #include #include "xls/codegen_v_1_5/block_conversion_pass.h" +#include "xls/codegen_v_1_5/block_conversion_wrapper_pass.h" #include "xls/codegen_v_1_5/block_finalization_pass.h" #include "xls/codegen_v_1_5/channel_to_port_io_lowering_pass.h" #include "xls/codegen_v_1_5/flow_control_insertion_pass.h" #include "xls/codegen_v_1_5/function_io_lowering_pass.h" #include "xls/codegen_v_1_5/idle_insertion_pass.h" #include "xls/codegen_v_1_5/pipeline_register_insertion_pass.h" +#include "xls/codegen_v_1_5/register_cleanup_pass.h" #include "xls/codegen_v_1_5/scheduled_block_conversion_pass.h" #include "xls/codegen_v_1_5/scheduling_pass.h" #include "xls/codegen_v_1_5/state_to_register_io_lowering_pass.h" +#include "xls/passes/dataflow_simplification_pass.h" +#include "xls/passes/dce_pass.h" +#include "xls/passes/identity_removal_pass.h" +#include "xls/passes/optimization_pass.h" namespace xls::codegen { -std::unique_ptr -CreateBlockConversionPassPipeline() { +std::unique_ptr CreateBlockConversionPassPipeline( + OptimizationContext& opt_context) { auto top = std::make_unique( "block_conversion", "Top level codegen v1.5 block conversion pipeline"); @@ -64,6 +70,20 @@ CreateBlockConversionPassPipeline() { // Lower scheduled block to standard block, inlining each stage. top->Add(); + // Clean up unused registers & load-enable bits (including flow-control + // registers). + top->Add(); + + // Clean up identity placeholders and unnecessary array/tuple manipulation. + top->Add(std::make_unique(), + opt_context); + top->Add( + std::make_unique(), opt_context); + + // Remove anything we created & then left dead. + top->Add( + std::make_unique(), opt_context); + return top; } diff --git a/xls/codegen_v_1_5/block_conversion_pass_pipeline.h b/xls/codegen_v_1_5/block_conversion_pass_pipeline.h index 7541593a34..37d20dcbeb 100644 --- a/xls/codegen_v_1_5/block_conversion_pass_pipeline.h +++ b/xls/codegen_v_1_5/block_conversion_pass_pipeline.h @@ -18,13 +18,14 @@ #include #include "xls/codegen_v_1_5/block_conversion_pass.h" +#include "xls/passes/optimization_pass.h" namespace xls::codegen { // Returns a pipeline which converts an unscheduled IR package into a standard // block. -std::unique_ptr -CreateBlockConversionPassPipeline(); +std::unique_ptr CreateBlockConversionPassPipeline( + OptimizationContext& opt_context); } // namespace xls::codegen diff --git a/xls/codegen_v_1_5/block_conversion_pass_pipeline_test.cc b/xls/codegen_v_1_5/block_conversion_pass_pipeline_test.cc index 5bff3c171c..a57a9fed72 100644 --- a/xls/codegen_v_1_5/block_conversion_pass_pipeline_test.cc +++ b/xls/codegen_v_1_5/block_conversion_pass_pipeline_test.cc @@ -13,8 +13,8 @@ // limitations under the License. #include +#include #include -#include #include #include #include @@ -54,7 +54,6 @@ #include "xls/common/status/matchers.h" #include "xls/common/status/ret_check.h" #include "xls/common/status/status_macros.h" -#include "xls/estimators/delay_model/delay_estimator.h" #include "xls/interpreter/block_evaluator.h" #include "xls/interpreter/block_interpreter.h" #include "xls/ir/bits.h" @@ -97,6 +96,7 @@ using ::testing::ElementsAre; using ::testing::Eq; using ::testing::Ge; using ::testing::HasSubstr; +using ::testing::IsEmpty; using ::testing::Optional; using ::testing::Pair; using ::testing::SizeIs; @@ -282,7 +282,7 @@ TEST_F(BlockConversionTest, SimpleFunction) { m::OutputPort("out", m::Add(m::InputPort("x"), m::InputPort("y")))); } -TEST_F(BlockConversionTest, DISABLED_SimpleFunctionWithNamedOutput) { +TEST_F(BlockConversionTest, SimpleFunctionWithNamedOutput) { auto p = CreatePackage(); FunctionBuilder fb(TestName(), p.get()); BValue x = fb.Param("x", p->GetBitsType(32)); @@ -331,7 +331,7 @@ TEST_F(BlockConversionTest, ZeroWidthInputsAndOutput) { EXPECT_EQ(block->GetPorts().size(), 4); } -TEST_F(BlockConversionTest, DISABLED_SimplePipelinedFunction) { +TEST_F(BlockConversionTest, SimplePipelinedFunction) { auto p = CreatePackage(); FunctionBuilder fb(TestName(), p.get()); BValue x = fb.Param("x", p->GetBitsType(32)); @@ -352,7 +352,7 @@ TEST_F(BlockConversionTest, DISABLED_SimplePipelinedFunction) { m::Add(m::InputPort("x"), m::InputPort("y")))))))); } -TEST_F(BlockConversionTest, DISABLED_TrivialPipelinedFunctionNoFlopping) { +TEST_F(BlockConversionTest, TrivialPipelinedFunctionNoFlopping) { auto p = CreatePackage(); FunctionBuilder fb(TestName(), p.get()); BValue x = fb.Param("x", p->GetBitsType(32)); @@ -373,7 +373,7 @@ TEST_F(BlockConversionTest, DISABLED_TrivialPipelinedFunctionNoFlopping) { m::Add(m::InputPort("x"), m::InputPort("y")))))))); } -TEST_F(BlockConversionTest, DISABLED_TrivialPipelinedFunctionFloppedInputs) { +TEST_F(BlockConversionTest, TrivialPipelinedFunctionFloppedInputs) { auto p = CreatePackage(); FunctionBuilder fb(TestName(), p.get()); BValue x = fb.Param("x", p->GetBitsType(32)); @@ -395,7 +395,7 @@ TEST_F(BlockConversionTest, DISABLED_TrivialPipelinedFunctionFloppedInputs) { m::Register(m::InputPort("x")), m::Register(m::InputPort("y"))))))))); } -TEST_F(BlockConversionTest, DISABLED_TrivialPipelinedFunctionFloppedOutputs) { +TEST_F(BlockConversionTest, TrivialPipelinedFunctionFloppedOutputs) { auto p = CreatePackage(); FunctionBuilder fb(TestName(), p.get()); BValue x = fb.Param("x", p->GetBitsType(32)); @@ -416,7 +416,7 @@ TEST_F(BlockConversionTest, DISABLED_TrivialPipelinedFunctionFloppedOutputs) { m::Add(m::InputPort("x"), m::InputPort("y"))))))))); } -TEST_F(BlockConversionTest, DISABLED_TrivialPipelinedFunctionFloppedIo) { +TEST_F(BlockConversionTest, TrivialPipelinedFunctionFloppedIo) { auto p = CreatePackage(); FunctionBuilder fb(TestName(), p.get()); BValue x = fb.Param("x", p->GetBitsType(32)); @@ -438,7 +438,7 @@ TEST_F(BlockConversionTest, DISABLED_TrivialPipelinedFunctionFloppedIo) { m::Register(m::InputPort("y")))))))))); } -TEST_F(BlockConversionTest, DISABLED_ZeroWidthPipeline) { +TEST_F(BlockConversionTest, ZeroWidthPipeline) { auto p = CreatePackage(); FunctionBuilder fb(TestName(), p.get()); BValue x = fb.Param("x", p->GetTupleType({})); @@ -454,7 +454,28 @@ TEST_F(BlockConversionTest, DISABLED_ZeroWidthPipeline) { "clk"), SchedulingOptions().pipeline_stages(3))); - EXPECT_EQ(block->GetRegisters().size(), 4); + EXPECT_THAT(block->GetRegisters(), IsEmpty()); +} + +TEST_F(BlockConversionTest, ZeroWidthPipelineWithValidControl) { + auto p = CreatePackage(); + FunctionBuilder fb(TestName(), p.get()); + BValue x = fb.Param("x", p->GetTupleType({})); + BValue y = fb.Param("y", p->GetBitsType(0)); + XLS_ASSERT_OK_AND_ASSIGN(Function * f, + fb.BuildWithReturnValue(fb.Tuple({x, y}))); + + XLS_ASSERT_OK_AND_ASSIGN( + Block * block, + ConvertToBlock(f, + CodegenOptions() + .flop_inputs(false) + .flop_outputs(false) + .clock_name("clk") + .valid_control("inputs_valid", "output_valid"), + SchedulingOptions().pipeline_stages(3))); + + EXPECT_THAT(block->GetRegisters(), SizeIs(2)); } // Verifies that an implicit token, as generated by the DSLX IR converter, is @@ -473,11 +494,10 @@ fn __itok__implicit_token__main(__token: token, __activated: bits[1]) -> fn __implicit_token__main() -> () { after_all.9: token = after_all(id=9) literal.10: bits[1] = literal(value=1, id=10) - invoke.11: (token, ()) = invoke(after_all.9, literal.10, - to_apply=__itok__implicit_token__main, id=11) tuple_index.12: token = - tuple_index(invoke.11, index=0, id=12) invoke.13: (token, ()) = - invoke(tuple_index.12, literal.10, to_apply=__itok__implicit_token__main, - id=13) ret tuple_index.14: () = tuple_index(invoke.13, index=1, id=14) + invoke.11: (token, ()) = invoke(after_all.9, literal.10, to_apply=__itok__implicit_token__main, id=11) + tuple_index.12: token = tuple_index(invoke.11, index=0, id=12) + invoke.13: (token, ()) = invoke(tuple_index.12, literal.10, to_apply=__itok__implicit_token__main, id=13) + ret tuple_index.14: () = tuple_index(invoke.13, index=1, id=14) } )"; XLS_ASSERT_OK_AND_ASSIGN(auto p, Parser::ParsePackage(kIrText)); @@ -489,7 +509,7 @@ fn __implicit_token__main() -> () { XLS_ASSERT_OK(VerifyBlock(block)); } -TEST_F(BlockConversionTest, DISABLED_SimpleProc) { +TEST_F(BlockConversionTest, SimpleProc) { const std::string ir_text = R"(package test chan in(bits[32], id=0, kind=single_value, ops=receive_only) @@ -516,7 +536,7 @@ proc my_proc(my_state: (), init={()}) { m::OutputPort("out", m::Neg(m::InputPort("in")))); } -TEST_F(BlockConversionTest, DISABLED_StreamingChannelMetadataForSimpleProc) { +TEST_F(BlockConversionTest, StreamingChannelMetadataForSimpleProc) { Package package(TestName()); Type* u32 = package.GetBitsType(32); XLS_ASSERT_OK_AND_ASSIGN( @@ -570,7 +590,7 @@ TEST_F(BlockConversionTest, DISABLED_StreamingChannelMetadataForSimpleProc) { IsOkAndHolds(Optional(m::InputPort("out_rdy")))); } -TEST_F(BlockConversionTest, DISABLED_SingleValueChannelMetadataForSimpleProc) { +TEST_F(BlockConversionTest, SingleValueChannelMetadataForSimpleProc) { Package package(TestName()); Type* u32 = package.GetBitsType(32); XLS_ASSERT_OK_AND_ASSIGN( @@ -624,7 +644,7 @@ TEST_F(BlockConversionTest, DISABLED_SingleValueChannelMetadataForSimpleProc) { IsOkAndHolds(std::nullopt)); } -TEST_F(BlockConversionTest, DISABLED_ProcWithVariousNextStateNodes) { +TEST_F(BlockConversionTest, ProcWithVariousNextStateNodes) { // A block with corner-case next state nodes (e.g., not dependent on state // param, same as state param, and shared next state nodes). auto p = CreatePackage(); @@ -712,7 +732,7 @@ TEST_F(BlockConversionTest, DISABLED_ProcWithVariousNextStateNodes) { IsOkAndHolds(ElementsAre(10, 20, 30))); } -TEST_F(BlockConversionTest, DISABLED_ProcWithNextStateNodeBeforeParam) { +TEST_F(BlockConversionTest, ProcWithNextStateNodeBeforeParam) { // A block with the next-state node scheduled before the param. auto p = CreatePackage(); Type* u32 = p->GetBitsType(32); @@ -795,7 +815,7 @@ TEST_F(BlockConversionTest, DISABLED_ProcWithNextStateNodeBeforeParam) { IsOkAndHolds(ElementsAre(10, 20, 30))); } -TEST_F(BlockConversionTest, DISABLED_ChannelDefaultSuffixName) { +TEST_F(BlockConversionTest, ChannelDefaultSuffixName) { const std::string ir_text = R"(package test chan in(bits[32], id=0, kind=streaming, ops=receive_only, @@ -850,7 +870,7 @@ proc my_proc(my_state: (), init={()}) { EXPECT_FALSE(HasNode("out2_vld", block_default_suffix)); } -TEST_F(BlockConversionTest, DISABLED_ChannelNonDefaultSuffixName) { +TEST_F(BlockConversionTest, ChannelNonDefaultSuffixName) { const std::string ir_text = R"(package test chan in(bits[32], id=0, kind=streaming, ops=receive_only, @@ -916,7 +936,7 @@ proc my_proc(my_state: (), init={()}) { EXPECT_FALSE(HasNode("out2_valid", block_nondefault_suffix)); } -TEST_F(BlockConversionTest, DISABLED_ProcWithMultipleInputChannels) { +TEST_F(BlockConversionTest, ProcWithMultipleInputChannels) { const std::string ir_text = R"(package test chan in0(bits[32], id=0, kind=single_value, ops=receive_only) @@ -961,7 +981,7 @@ proc my_proc(my_state: (), init={()}) { m::InputPort("in0")))); } -TEST_F(BlockConversionTest, DISABLED_OnlyFIFOOutProc) { +TEST_F(BlockConversionTest, OnlyFIFOOutProc) { const std::string ir_text = R"(package test chan in(bits[32], id=0, kind=single_value, ops=receive_only) chan out(bits[32], id=1, kind=streaming, ops=send_only, flow_control=ready_valid) @@ -987,10 +1007,10 @@ proc my_proc(st: (), init={()}) { EXPECT_THAT(FindNode("out", block), m::OutputPort("out", m::InputPort("in"))); EXPECT_THAT(FindNode("out_vld", block), m::OutputPort("out_vld", m::And(m::Literal(1), m::Literal(1), - m::Literal(1), m::Literal(1)))); + m::Literal(1)))); } -TEST_F(BlockConversionTest, DISABLED_NoRegsIfChannelsHaveNoFlopsSet) { +TEST_F(BlockConversionTest, NoRegsIfChannelsHaveNoFlopsSet) { constexpr std::string_view kIrText = R"( package my_package @@ -1024,7 +1044,7 @@ top proc my_proc() { EXPECT_THAT(block->GetRegisters(), testing::IsEmpty()); } -TEST_F(BlockConversionTest, DISABLED_OnlyFIFOInProcGateRecvsTrue) { +TEST_F(BlockConversionTest, OnlyFIFOInProcGateRecvsTrue) { const std::string ir_text = R"(package test chan in(bits[32], id=0, kind=streaming, ops=receive_only, flow_control=ready_valid) chan out(bits[32], id=1, kind=single_value, ops=send_only) @@ -1055,11 +1075,17 @@ proc my_proc(st: (), init={()}) { {m::Literal(0), m::InputPort("in")}))); EXPECT_THAT(FindNode("in", block), m::InputPort("in")); EXPECT_THAT(FindNode("in_vld", block), m::InputPort("in_vld")); - EXPECT_THAT(FindNode("in_rdy", block), - m::OutputPort("in_rdy", m::And(m::Literal(1), m::Literal(1)))); + EXPECT_THAT( + FindNode("in_rdy", block), + m::OutputPort( + "in_rdy", + m::And(m::And(m::Literal(1), + m::And(m::Literal(1), m::Or(m::InputPort("in_vld"), + m::Not(m::Literal(1))))), + m::Literal(1), m::Literal(1)))); } -TEST_F(BlockConversionTest, DISABLED_OnlyFIFOInProcGateRecvsFalse) { +TEST_F(BlockConversionTest, OnlyFIFOInProcGateRecvsFalse) { const std::string ir_text = R"(package test chan in(bits[32], id=0, kind=streaming, ops=receive_only, flow_control=ready_valid) chan out(bits[32], id=1, kind=single_value, ops=send_only) @@ -1087,11 +1113,17 @@ proc my_proc(st: (), init={()}) { EXPECT_THAT(FindNode("out", block), m::OutputPort("out", m::InputPort("in"))); EXPECT_THAT(FindNode("in", block), m::InputPort("in")); EXPECT_THAT(FindNode("in_vld", block), m::InputPort("in_vld")); - EXPECT_THAT(FindNode("in_rdy", block), - m::OutputPort("in_rdy", m::And(m::Literal(1), m::Literal(1)))); + EXPECT_THAT( + FindNode("in_rdy", block), + m::OutputPort( + "in_rdy", + m::And(m::And(m::Literal(1), + m::And(m::Literal(1), m::Or(m::InputPort("in_vld"), + m::Not(m::Literal(1))))), + m::Literal(1), m::Literal(1)))); } -TEST_F(BlockConversionTest, DISABLED_UnconditionalSendRdyVldProc) { +TEST_F(BlockConversionTest, UnconditionalSendRdyVldProc) { const std::string ir_text = R"(package test chan in(bits[32], id=0, kind=single_value, ops=receive_only) chan out(bits[32], id=1, kind=streaming, ops=send_only, flow_control=ready_valid) @@ -1116,13 +1148,12 @@ proc my_proc(st: (), init={()}) { EXPECT_THAT(FindNode("out", block), m::OutputPort("out", m::InputPort("in"))); EXPECT_THAT(FindNode("out_vld", block), - m::OutputPort("out_vld", m::And(m::Literal(1), m::Literal(1), - m::Literal(1)))); + m::OutputPort("out_vld", m::And(m::Literal(1), m::Literal(1)))); EXPECT_THAT(FindNode("out_rdy", block), m::InputPort("out_rdy")); } // Ensure that the output of the receive is zero when the predicate is false. -TEST_F(BlockConversionTest, DISABLED_ReceiveIfIsZeroWhenPredicateIsFalse) { +TEST_F(BlockConversionTest, ReceiveIfIsZeroWhenPredicateIsFalse) { Package package(TestName()); Type* u32 = package.GetBitsType(32); XLS_ASSERT_OK_AND_ASSIGN( @@ -1165,8 +1196,7 @@ TEST_F(BlockConversionTest, DISABLED_ReceiveIfIsZeroWhenPredicateIsFalse) { // Ensure that the output of the receive is passthrough when the predicate is // false. -TEST_F(BlockConversionTest, - DISABLED_ReceiveIfIsPassthroughWhenPredicateIsFalse) { +TEST_F(BlockConversionTest, ReceiveIfIsPassthroughWhenPredicateIsFalse) { Package package(TestName()); Type* u32 = package.GetBitsType(32); XLS_ASSERT_OK_AND_ASSIGN( @@ -1209,7 +1239,7 @@ TEST_F(BlockConversionTest, } // Ensure that the output of the receive is zero when the data is not valid. -TEST_F(BlockConversionTest, DISABLED_NonblockingReceiveIsZeroWhenDataInvalid) { +TEST_F(BlockConversionTest, NonblockingReceiveIsZeroWhenDataInvalid) { Package package(TestName()); Type* u32 = package.GetBitsType(32); XLS_ASSERT_OK_AND_ASSIGN( @@ -1247,8 +1277,7 @@ TEST_F(BlockConversionTest, DISABLED_NonblockingReceiveIsZeroWhenDataInvalid) { // Ensure that the output of the receive is passthrough when the data is not // valid. -TEST_F(BlockConversionTest, - DISABLED_NonblockingReceiveIsPassthroughWhenDataInvalid) { +TEST_F(BlockConversionTest, NonblockingReceiveIsPassthroughWhenDataInvalid) { Package package(TestName()); Type* u32 = package.GetBitsType(32); XLS_ASSERT_OK_AND_ASSIGN( @@ -1287,8 +1316,7 @@ TEST_F(BlockConversionTest, } // Ensure that the output of the receive is zero when the predicate is false. -TEST_F(BlockConversionTest, - DISABLED_NonblockingReceiveIsZeroWhenPredicateIsFalse) { +TEST_F(BlockConversionTest, NonblockingReceiveIsZeroWhenPredicateIsFalse) { Package package(TestName()); Type* u32 = package.GetBitsType(32); XLS_ASSERT_OK_AND_ASSIGN( @@ -1336,7 +1364,7 @@ TEST_F(BlockConversionTest, // Ensure that the output of the receive is passthrough when the predicate is // false. TEST_F(BlockConversionTest, - DISABLED_NonblockingReceiveIsPassthroughWhenPredicateIsFalse) { + NonblockingReceiveIsPassthroughWhenPredicateIsFalse) { Package package(TestName()); Type* u32 = package.GetBitsType(32); XLS_ASSERT_OK_AND_ASSIGN( @@ -1382,7 +1410,7 @@ TEST_F(BlockConversionTest, Pair("out", 42)))); } -TEST_F(BlockConversionTest, DISABLED_TwoToOneProc) { +TEST_F(BlockConversionTest, TwoToOneProc) { Package package(TestName()); Type* u32 = package.GetBitsType(32); XLS_ASSERT_OK_AND_ASSIGN( @@ -1455,10 +1483,44 @@ TEST_F(BlockConversionTest, DISABLED_TwoToOneProc) { {"b_vld", 1}, {"out_rdy", 1}}), IsOkAndHolds(UnorderedElementsAre(Pair("out_vld", 0), Pair("b_rdy", 0), - Pair("out", 123), Pair("a_rdy", 1)))); + Pair("out", 123), Pair("a_rdy", 0)))); +} + +TEST_F(BlockConversionTest, JoinProc) { + Package package(TestName()); + Type* u32 = package.GetBitsType(32); + XLS_ASSERT_OK_AND_ASSIGN( + Channel * ch_a, + package.CreateStreamingChannel("a", ChannelOps::kReceiveOnly, u32)); + XLS_ASSERT_OK_AND_ASSIGN( + Channel * ch_b, + package.CreateStreamingChannel("b", ChannelOps::kReceiveOnly, u32)); + XLS_ASSERT_OK_AND_ASSIGN( + Channel * ch_out, + package.CreateStreamingChannel("out", ChannelOps::kSendOnly, u32)); + + TokenlessProcBuilder pb(TestName(), /*token_name=*/"tkn", &package); + BValue a = pb.Receive(ch_a); + BValue b = pb.Receive(ch_b); + pb.Send(ch_out, pb.Add(a, b)); + + XLS_ASSERT_OK_AND_ASSIGN(Proc * proc, pb.Build({})); + + XLS_ASSERT_OK_AND_ASSIGN( + Block * block, + ConvertToBlock(proc, codegen_options().generate_combinational(true))); + + // A is valid, C is ready, B is not yet valid. + // a_rdy must be 0 to avoid accidentally consuming from A and losing data. + EXPECT_THAT( + InterpretCombinationalBlock( + block, + {{"a", 10}, {"a_vld", 1}, {"b", 20}, {"b_vld", 0}, {"out_rdy", 1}}), + IsOkAndHolds(UnorderedElementsAre(Pair("out_vld", 0), Pair("a_rdy", 0), + Pair("b_rdy", 0), Pair("out", 30)))); } -TEST_F(BlockConversionTest, DISABLED_OneToTwoProc) { +TEST_F(BlockConversionTest, OneToTwoProc) { Package package(TestName()); Type* u32 = package.GetBitsType(32); XLS_ASSERT_OK_AND_ASSIGN( @@ -1511,7 +1573,7 @@ TEST_F(BlockConversionTest, DISABLED_OneToTwoProc) { block, {{"dir", 1}, {"in", 123}, {"in_vld", 0}, {"a_rdy", 1}, {"b_rdy", 1}}), IsOkAndHolds(UnorderedElementsAre(Pair("a", 123), Pair("b_vld", 0), - Pair("in_rdy", 1), Pair("a_vld", 0), + Pair("in_rdy", 0), Pair("a_vld", 0), Pair("b", 123)))); // Output A selected. Input valid and output ready *not* asserted. @@ -1524,7 +1586,7 @@ TEST_F(BlockConversionTest, DISABLED_OneToTwoProc) { Pair("b", 123)))); } -TEST_F(BlockConversionTest, DISABLED_RamChannelsAreNotFlopped) { +TEST_F(BlockConversionTest, RamChannelsAreNotFlopped) { // Test that RAM channels are not flopped even when flop_inputs/outputs is set auto package = CreatePackage(); Type* u32 = package->GetBitsType(32); @@ -1611,7 +1673,7 @@ TEST_F(BlockConversionTest, DISABLED_RamChannelsAreNotFlopped) { EXPECT_FALSE(has_reg_with_substr(block, "ram_wr_comp")); } -TEST_F(BlockConversionTest, DISABLED_FlopSingleValueChannelProc) { +TEST_F(BlockConversionTest, FlopSingleValueChannelProc) { const std::string ir_text = R"(package test chan in(bits[32], id=0, kind=single_value, ops=receive_only) chan out(bits[32], id=1, kind=single_value, ops=send_only) @@ -1643,6 +1705,10 @@ proc my_proc(tkn: token, st: (), init={token, ()}) { options.reset("rst_n", false, /*active_low=*/true, false); { + XLS_ASSERT_OK_AND_ASSIGN(std::unique_ptr p, + ClonePackage(package.get())); + XLS_ASSERT_OK_AND_ASSIGN(Proc * proc, p->GetProc("my_proc")); + options.flop_single_value_channels(true).module_name( "with_single_value_channel"); @@ -1660,6 +1726,10 @@ proc my_proc(tkn: token, st: (), init={token, ()}) { } { + XLS_ASSERT_OK_AND_ASSIGN(std::unique_ptr p, + ClonePackage(package.get())); + XLS_ASSERT_OK_AND_ASSIGN(Proc * proc, p->GetProc("my_proc")); + options.flop_single_value_channels(false).module_name( "no_single_value_channel"); @@ -1717,7 +1787,7 @@ class SimplePipelinedProcTest : public ProcConversionTestFixture { } }; -TEST_F(SimplePipelinedProcTest, DISABLED_ChannelDefaultSuffixName) { +TEST_F(SimplePipelinedProcTest, ChannelDefaultSuffixName) { CodegenOptions options; options.flop_inputs(false).flop_outputs(false).clock_name("clk"); options.valid_control("input_valid", "output_valid"); @@ -1737,7 +1807,7 @@ TEST_F(SimplePipelinedProcTest, DISABLED_ChannelDefaultSuffixName) { EXPECT_TRUE(HasNode("out_vld", block_default_suffix)); } -TEST_F(SimplePipelinedProcTest, DISABLED_ChannelNonDefaultSuffixName) { +TEST_F(SimplePipelinedProcTest, ChannelNonDefaultSuffixName) { CodegenOptions options; options.flop_inputs(false) .flop_outputs(false) @@ -1762,7 +1832,7 @@ TEST_F(SimplePipelinedProcTest, DISABLED_ChannelNonDefaultSuffixName) { EXPECT_TRUE(HasNode("out_valid", block_nondefault_suffix)); } -TEST_F(SimplePipelinedProcTest, DISABLED_BasicDatapathResetAndInputFlop) { +TEST_F(SimplePipelinedProcTest, BasicDatapathResetAndInputFlop) { CodegenOptions options; options.flop_inputs(true).flop_outputs(false).clock_name("clk"); options.valid_control("input_valid", "output_valid"); @@ -1850,11 +1920,12 @@ TEST_F(SimplePipelinedProcTest, DISABLED_BasicDatapathResetAndInputFlop) { /*column_width=*/10, inputs, outputs, expected_outputs)); for (int64_t i = 0; i < outputs.size(); ++i) { - EXPECT_EQ(outputs.at(i), expected_outputs.at(i)); + EXPECT_EQ(outputs.at(i), expected_outputs.at(i)) + << "Mismatch at cycle " << i; } } -TEST_F(SimplePipelinedProcTest, DISABLED_BasicResetAndStall) { +TEST_F(SimplePipelinedProcTest, BasicResetAndStall) { CodegenOptions options; options.flop_inputs(false).flop_outputs(false).clock_name("clk"); options.valid_control("input_valid", "output_valid"); @@ -1882,17 +1953,12 @@ TEST_F(SimplePipelinedProcTest, DISABLED_BasicResetAndStall) { 0, 9, {{"rst", 1}, {"in_vld", 1}, {"out_rdy", 1}}, inputs)); XLS_ASSERT_OK(SetSignalsOverCycles( 0, 2, {{"in_rdy", 1}, {"out_vld", 0}, {"out", 0}}, expected_outputs)); - XLS_ASSERT_OK(SetSignalsOverCycles(1, 1, {{"out", 0}}, expected_outputs)); + XLS_ASSERT_OK(SetSignalsOverCycles(1, 2, {{"out", 0}}, expected_outputs)); XLS_ASSERT_OK(SetSignalsOverCycles(3, 9, {{"in_rdy", 1}, {"out_vld", 0}}, expected_outputs)); XLS_ASSERT_OK_AND_ASSIGN(running_out_val, SetIncrementingSignalOverCycles( 3, 9, "out", running_out_val, expected_outputs)); - // The way SDC schedules this requires this, because there's a not node after - // a register in stage 0, so on reset that outputs !0 = INT_MAX. - // We can't easily change SimplePipelinedProcTest to use a manual schedule - // because it accepts the number of stages as a parameter. - expected_outputs.at(2).at("out") = std::numeric_limits::max(); // Once reset is deasserted, then the pipeline is closed, no further changes // in the output is expected if the input is not valid. @@ -2062,7 +2128,7 @@ class SimplePipelinedProcTestSweepFixture } }; -TEST_P(SimplePipelinedProcTestSweepFixture, DISABLED_RandomStalling) { +TEST_P(SimplePipelinedProcTestSweepFixture, RandomStalling) { int64_t stage_count = std::get<0>(GetParam()); bool flop_inputs = std::get<1>(GetParam()); bool flop_outputs = std::get<2>(GetParam()); @@ -2263,7 +2329,7 @@ class SimpleRunningCounterProcTestSweepFixture } }; -TEST_P(SimpleRunningCounterProcTestSweepFixture, DISABLED_RandomStalling) { +TEST_P(SimpleRunningCounterProcTestSweepFixture, RandomStalling) { int64_t stage_count = std::get<0>(GetParam()); bool active_low_reset = std::get<1>(GetParam()); bool flop_inputs = std::get<2>(GetParam()); @@ -2410,7 +2476,7 @@ INSTANTIATE_TEST_SUITE_P( CodegenOptions::IOKind::kZeroLatencyBuffer)), SimpleRunningCounterProcTestSweepFixture::PrintToStringParamName); -// Fixture used to test pipelined BlockConversion on a multi input block. +// Fixture used to test pipelined BlockConversion on a multi-input block. class MultiInputPipelinedProcTest : public ProcConversionTestFixture { protected: absl::StatusOr> BuildBlockInPackage( @@ -2479,7 +2545,7 @@ class MultiInputPipelinedProcTestSweepFixture } }; -TEST_P(MultiInputPipelinedProcTestSweepFixture, DISABLED_RandomStalling) { +TEST_P(MultiInputPipelinedProcTestSweepFixture, RandomStalling) { int64_t stage_count = std::get<0>(GetParam()); bool flop_inputs = std::get<1>(GetParam()); bool flop_outputs = std::get<2>(GetParam()); @@ -2590,10 +2656,11 @@ TEST_P(MultiInputPipelinedProcTestSweepFixture, DISABLED_RandomStalling) { {reset_signal, SignalType::kInput, active_low_reset}, inputs, outputs)); - EXPECT_TRUE(output_sequence.size() == input0_sequence.size() || - output_sequence.size() + 1 == input0_sequence.size()); - EXPECT_TRUE(output_sequence.size() == input1_sequence.size() || - output_sequence.size() + 1 == input1_sequence.size()); + EXPECT_LE(output_sequence.size(), input0_sequence.size()); + EXPECT_GE(output_sequence.size() + 1, input1_sequence.size()); + + EXPECT_LE(output_sequence.size(), input1_sequence.size()); + EXPECT_GE(output_sequence.size() + 1, input0_sequence.size()); for (int64_t i = 0; i < output_sequence.size(); ++i) { int64_t in0_val = input0_sequence.at(i).value; @@ -2623,7 +2690,7 @@ INSTANTIATE_TEST_SUITE_P( class SpecificIoKindsTest : public ProcConversionTestFixture, public testing::WithParamInterface {}; -TEST_P(SpecificIoKindsTest, DISABLED_InputChannelSpecificFlopKindsRespected) { +TEST_P(SpecificIoKindsTest, InputChannelSpecificFlopKindsRespected) { // Compile once with a specific override for the channel and once with the // default set and compare outputs. auto p = CreatePackage(); @@ -2688,7 +2755,7 @@ TEST_P(SpecificIoKindsTest, DISABLED_InputChannelSpecificFlopKindsRespected) { EXPECT_EQ(p->blocks().front()->DumpIr(), p2->blocks().front()->DumpIr()); } -TEST_P(SpecificIoKindsTest, DISABLED_InputChannelDefaultFlopKindsChange) { +TEST_P(SpecificIoKindsTest, InputChannelDefaultFlopKindsChange) { // Compile once with a specific override for the channel and once with the // default set and compare outputs. auto p = CreatePackage(); @@ -2769,7 +2836,7 @@ TEST_P(SpecificIoKindsTest, DISABLED_InputChannelDefaultFlopKindsChange) { EXPECT_NE(p->blocks().front()->DumpIr(), p2->blocks().front()->DumpIr()); } -TEST_P(SpecificIoKindsTest, DISABLED_OutputChannelSpecificFlopKindsRespected) { +TEST_P(SpecificIoKindsTest, OutputChannelSpecificFlopKindsRespected) { // Compile once with a specific override for the channel and once with the // default set and compare outputs. auto p = CreatePackage(); @@ -2840,7 +2907,7 @@ TEST_P(SpecificIoKindsTest, DISABLED_OutputChannelSpecificFlopKindsRespected) { EXPECT_EQ(p->blocks().front()->DumpIr(), p2->blocks().front()->DumpIr()); } -TEST_P(SpecificIoKindsTest, DISABLED_OutputChannelDefaultFlopKindsChange) { +TEST_P(SpecificIoKindsTest, OutputChannelDefaultFlopKindsChange) { // Compile once with a specific override for the channel and once with the // default set and compare outputs. auto p = CreatePackage(); @@ -2928,7 +2995,206 @@ INSTANTIATE_TEST_SUITE_P(SpecificIoKindsTest, SpecificIoKindsTest, FlopKind::kNone), testing::PrintToStringParamName()); -TEST_F(MultiInputPipelinedProcTest, DISABLED_IdleSignalNoFlops) { +TEST_F(MultiInputPipelinedProcTest, IdleSignalNoFlops) { + int64_t stage_count = 4; + bool active_low_reset = true; + + CodegenOptions options; + options.flop_inputs(false).flop_outputs(false).clock_name("clk"); + options.valid_control("input_valid", "output_valid"); + options.reset("rst_n", false, /*active_low=*/active_low_reset, false); + options.add_idle_output(true); + + XLS_ASSERT_OK_AND_ASSIGN( + std::unique_ptr package, + BuildBlockInPackage(/*stage_count=*/stage_count, options)); + + XLS_ASSERT_OK_AND_ASSIGN(Block * block, package->GetBlock(kBlockName)); + + VLOG(2) << "Multi input counting pipelined block"; + XLS_VLOG_LINES(2, block->DumpIr()); + + // The input stimulus to this test are + // 8. 10 cycles of idle + // 9. 10 cycles of data on in0 and in1 + std::vector> inputs; + std::vector> expected_outputs; + + // During reset, the output will be invalid, but the pipeline + // is open and the in data will flow through to the output. + + // 1. 10 cycles of reset - idle will be 1 + XLS_ASSERT_OK(SetSignalsOverCycles( + 0, 9, {{"rst_n", 0}, {"in0_vld", 0}, {"in1_vld", 0}, {"out_rdy", 1}}, + inputs)); + XLS_ASSERT_OK(SetSignalsOverCycles( + 0, 9, {{"in0_rdy", 0}, {"in1_rdy", 0}, {"out_vld", 0}, {"idle", 1}}, + expected_outputs)); + + // 2. 10 cycles of idle - idle remains 1 + XLS_ASSERT_OK(SetSignalsOverCycles( + 10, 19, {{"rst_n", 1}, {"in0_vld", 0}, {"in1_vld", 0}, {"out_rdy", 1}}, + inputs)); + XLS_ASSERT_OK(SetSignalsOverCycles( + 10, 19, {{"in0_rdy", 0}, {"in1_rdy", 0}, {"out_vld", 0}, {"idle", 1}}, + expected_outputs)); + + // 3. 20 cycles of data on in0 - idle remains 1 since the pipeline is still + // waiting on in1. + XLS_ASSERT_OK(SetSignalsOverCycles( + 10, 29, {{"rst_n", 1}, {"in0_vld", 1}, {"in1_vld", 0}, {"out_rdy", 1}}, + inputs)); + XLS_ASSERT_OK(SetSignalsOverCycles( + 10, 29, {{"in0_rdy", 0}, {"in1_rdy", 0}, {"out_vld", 0}, {"idle", 1}}, + expected_outputs)); + + // 4. 1 cycle of data on in1 - allows 4-stage pipeline to drain + // 5. After 4 more cycles (on 34th cycle), pipeline drains and block becomes + // idle + XLS_ASSERT_OK(SetSignalsOverCycles( + 30, 30, {{"rst_n", 1}, {"in0_vld", 1}, {"in1_vld", 1}, {"out_rdy", 1}}, + inputs)); + XLS_ASSERT_OK(SetSignalsOverCycles( + 30, 30, {{"in0_rdy", 1}, {"in1_rdy", 1}, {"out_vld", 0}, {"idle", 0}}, + expected_outputs)); + XLS_ASSERT_OK(SetSignalsOverCycles( + 31, 39, {{"rst_n", 1}, {"in0_vld", 0}, {"in1_vld", 0}, {"out_rdy", 1}}, + inputs)); + XLS_ASSERT_OK(SetSignalsOverCycles( + 31, 32, {{"in0_rdy", 0}, {"in1_rdy", 0}, {"out_vld", 0}, {"idle", 0}}, + expected_outputs)); + XLS_ASSERT_OK(SetSignalsOverCycles( + 33, 33, {{"in0_rdy", 0}, {"in1_rdy", 0}, {"out_vld", 1}, {"idle", 0}}, + expected_outputs)); + XLS_ASSERT_OK(SetSignalsOverCycles( + 34, 39, {{"in0_rdy", 0}, {"in1_rdy", 0}, {"out_vld", 0}, {"idle", 1}}, + expected_outputs)); + + // 6. 30 cycles of data on in1 - idle remains 1 since the pipeline is + // still waiting on in0. + XLS_ASSERT_OK(SetSignalsOverCycles( + 40, 69, {{"rst_n", 1}, {"in0_vld", 0}, {"in1_vld", 1}, {"out_rdy", 1}}, + inputs)); + XLS_ASSERT_OK(SetSignalsOverCycles( + 40, 69, {{"in0_rdy", 0}, {"in1_rdy", 0}, {"out_vld", 0}, {"idle", 1}}, + expected_outputs)); + + // 7. 1 cycle of data on in0 - allows 4-stage pipeline to drain + // 8. After 4 more cycles (on 74th cycle), pipeline drains and block becomes + // idle + XLS_ASSERT_OK(SetSignalsOverCycles( + 70, 70, {{"rst_n", 1}, {"in0_vld", 1}, {"in1_vld", 1}, {"out_rdy", 1}}, + inputs)); + XLS_ASSERT_OK(SetSignalsOverCycles( + 70, 70, {{"in0_rdy", 1}, {"in1_rdy", 1}, {"out_vld", 0}, {"idle", 0}}, + expected_outputs)); + XLS_ASSERT_OK(SetSignalsOverCycles( + 71, 79, {{"rst_n", 1}, {"in0_vld", 0}, {"in1_vld", 0}, {"out_rdy", 1}}, + inputs)); + XLS_ASSERT_OK(SetSignalsOverCycles( + 71, 72, {{"in0_rdy", 0}, {"in1_rdy", 0}, {"out_vld", 0}, {"idle", 0}}, + expected_outputs)); + XLS_ASSERT_OK(SetSignalsOverCycles( + 73, 73, {{"in0_rdy", 0}, {"in1_rdy", 0}, {"out_vld", 1}, {"idle", 0}}, + expected_outputs)); + XLS_ASSERT_OK(SetSignalsOverCycles( + 74, 79, {{"in0_rdy", 0}, {"in1_rdy", 0}, {"out_vld", 0}, {"idle", 1}}, + expected_outputs)); + + // 9. Skipping two cycles of valid input data, then drain the pipeline + // input on cycle 80 appears on the output on cycle 83 + // input on cycle 83 appears on the output on cycle 86 + // idle aserts on cycle 87 + XLS_ASSERT_OK(SetSignalsOverCycles( + 80, 80, {{"rst_n", 1}, {"in0_vld", 1}, {"in1_vld", 1}, {"out_rdy", 1}}, + inputs)); + XLS_ASSERT_OK(SetSignalsOverCycles( + 80, 80, {{"in0_rdy", 1}, {"in1_rdy", 1}, {"out_vld", 0}, {"idle", 0}}, + expected_outputs)); + + XLS_ASSERT_OK(SetSignalsOverCycles( + 81, 82, {{"rst_n", 1}, {"in0_vld", 0}, {"in1_vld", 0}, {"out_rdy", 1}}, + inputs)); + XLS_ASSERT_OK(SetSignalsOverCycles( + 81, 82, {{"in0_rdy", 0}, {"in1_rdy", 0}, {"out_vld", 0}, {"idle", 0}}, + expected_outputs)); + + XLS_ASSERT_OK(SetSignalsOverCycles( + 83, 83, {{"rst_n", 1}, {"in0_vld", 1}, {"in1_vld", 1}, {"out_rdy", 1}}, + inputs)); + XLS_ASSERT_OK(SetSignalsOverCycles( + 83, 83, {{"in0_rdy", 1}, {"in1_rdy", 1}, {"out_vld", 1}, {"idle", 0}}, + expected_outputs)); + + XLS_ASSERT_OK(SetSignalsOverCycles( + 84, 89, {{"rst_n", 1}, {"in0_vld", 0}, {"in1_vld", 0}, {"out_rdy", 1}}, + inputs)); + XLS_ASSERT_OK(SetSignalsOverCycles( + 84, 85, {{"in0_rdy", 0}, {"in1_rdy", 0}, {"out_vld", 0}, {"idle", 0}}, + expected_outputs)); + XLS_ASSERT_OK(SetSignalsOverCycles( + 86, 86, {{"in0_rdy", 0}, {"in1_rdy", 0}, {"out_vld", 1}, {"idle", 0}}, + expected_outputs)); + XLS_ASSERT_OK(SetSignalsOverCycles( + 87, 89, {{"in0_rdy", 0}, {"in1_rdy", 0}, {"out_vld", 0}, {"idle", 1}}, + expected_outputs)); + + // 12. Continuous data for 10 cycles means that idle becomes 0 again. + // input on cycle 90 appears on the output on cycle 93 + XLS_ASSERT_OK(SetSignalsOverCycles( + 90, 99, {{"rst_n", 1}, {"in0_vld", 1}, {"in1_vld", 1}, {"out_rdy", 1}}, + inputs)); + XLS_ASSERT_OK(SetSignalsOverCycles( + 90, 92, {{"in0_rdy", 1}, {"in1_rdy", 1}, {"out_vld", 0}, {"idle", 0}}, + expected_outputs)); + XLS_ASSERT_OK(SetSignalsOverCycles( + 93, 99, {{"in0_rdy", 1}, {"in1_rdy", 1}, {"out_vld", 1}, {"idle", 0}}, + expected_outputs)); + + // Fill in the input data + uint64_t running_in_val = 0; + XLS_ASSERT_OK(SetIncrementingSignalOverCycles(0, inputs.size() - 1, "in0", + running_in_val, inputs)); + XLS_ASSERT_OK(SetIncrementingSignalOverCycles(0, inputs.size() - 1, "in1", + running_in_val, inputs)); + + // Run interpreter + std::vector> outputs; + XLS_ASSERT_OK_AND_ASSIGN(outputs, InterpretSequentialBlock(block, inputs)); + + // Add a cycle count for easier comparison with simulation results. + XLS_ASSERT_OK(SetIncrementingSignalOverCycles(0, outputs.size() - 1, "cycle", + 0, outputs)); + XLS_ASSERT_OK(SetIncrementingSignalOverCycles(0, expected_outputs.size() - 1, + "cycle", 0, expected_outputs)); + + XLS_ASSERT_OK(VLogTestPipelinedIO( + std::vector{{"cycle", SignalType::kOutput}, + {"rst_n", SignalType::kInput, true}, + {"in0", SignalType::kInput}, + {"in0_vld", SignalType::kInput}, + {"in0_rdy", SignalType::kOutput}, + {"in1", SignalType::kInput}, + {"in1_vld", SignalType::kInput}, + {"in1_rdy", SignalType::kOutput}, + {"out", SignalType::kOutput}, + {"out_vld", SignalType::kOutput}, + {"out_rdy", SignalType::kInput}, + {"idle", SignalType::kOutput}}, + /*column_width=*/10, inputs, outputs)); + + ASSERT_EQ(inputs.size(), expected_outputs.size()); + ASSERT_EQ(outputs.size(), expected_outputs.size()); + + for (int64_t i = 0; i < outputs.size(); ++i) { + // ignore the actual value of the output + outputs[i].erase("out"); + + EXPECT_EQ(outputs.at(i), expected_outputs.at(i)); + } +} + +TEST_F(MultiInputPipelinedProcTest, IdleSignalInputOutputFlops) { int64_t stage_count = 4; bool active_low_reset = true; @@ -2964,7 +3230,7 @@ TEST_F(MultiInputPipelinedProcTest, DISABLED_IdleSignalNoFlops) { 0, 9, {{"in0_rdy", 0}, {"in1_rdy", 0}, {"out_vld", 0}, {"idle", 1}}, expected_outputs)); - // 2. 10 cycles of idle -- idle remains 1 + // 2. 10 cycles of idle - idle remains 1 XLS_ASSERT_OK(SetSignalsOverCycles( 10, 19, {{"rst_n", 1}, {"in0_vld", 0}, {"in1_vld", 0}, {"out_rdy", 1}}, inputs)); @@ -2972,10 +3238,9 @@ TEST_F(MultiInputPipelinedProcTest, DISABLED_IdleSignalNoFlops) { 10, 19, {{"in0_rdy", 0}, {"in1_rdy", 0}, {"out_vld", 0}, {"idle", 1}}, expected_outputs)); - // 3. 1 cycle of data on in0 - idle immediately becomes 0 due to - // combinational path - // 4. 20 cycles of idle - idle continues to remain 0 as the pipeline is not - // flowing. + // 3. 1 cycle of data on in0 - we receive from in0 into the input flop & + // briefly see idle go to 0. + // 4. 20 cycles of idle - idle goes back to 1 while we wait for in1. XLS_ASSERT_OK(SetSignalsOverCycles( 10, 10, {{"rst_n", 1}, {"in0_vld", 1}, {"in1_vld", 0}, {"out_rdy", 1}}, inputs)); @@ -2986,12 +3251,13 @@ TEST_F(MultiInputPipelinedProcTest, DISABLED_IdleSignalNoFlops) { 11, 29, {{"rst_n", 1}, {"in0_vld", 0}, {"in1_vld", 0}, {"out_rdy", 1}}, inputs)); XLS_ASSERT_OK(SetSignalsOverCycles( - 11, 29, {{"in0_rdy", 0}, {"in1_rdy", 0}, {"out_vld", 0}, {"idle", 0}}, + 11, 29, {{"in0_rdy", 0}, {"in1_rdy", 0}, {"out_vld", 0}, {"idle", 1}}, expected_outputs)); - // 5. 1 cycle of data on in1 -- allows 4-stage pipeline to drain + // 5. 1 cycle of data on in1 - allows 4-stage pipeline to drain, idle + // goes low immediately // 6. After 5 more cycles (on 36th cycle), pipeline drains and block becomes - // idle + // idle XLS_ASSERT_OK(SetSignalsOverCycles( 30, 30, {{"rst_n", 1}, {"in0_vld", 0}, {"in1_vld", 1}, {"out_rdy", 1}}, inputs)); @@ -3011,10 +3277,9 @@ TEST_F(MultiInputPipelinedProcTest, DISABLED_IdleSignalNoFlops) { 36, 39, {{"in0_rdy", 0}, {"in1_rdy", 0}, {"out_vld", 0}, {"idle", 1}}, expected_outputs)); - // 7. 1 cycle of data on in1 - idle immediately becomes 0 due to - // combinational path - // 8. 20 cycles of idle - idle continues to remain 0 as the pipeline is not - // flowing. + // 7. 1 cycle of data on in1 - we receive from in1 into the input flop & + // briefly see idle go to 0. + // 8. 20 cycles of idle - idle goes back to 1 while we wait for in0. XLS_ASSERT_OK(SetSignalsOverCycles( 40, 40, {{"rst_n", 1}, {"in0_vld", 0}, {"in1_vld", 1}, {"out_rdy", 1}}, inputs)); @@ -3025,12 +3290,13 @@ TEST_F(MultiInputPipelinedProcTest, DISABLED_IdleSignalNoFlops) { 41, 69, {{"rst_n", 1}, {"in0_vld", 0}, {"in1_vld", 0}, {"out_rdy", 1}}, inputs)); XLS_ASSERT_OK(SetSignalsOverCycles( - 41, 69, {{"in0_rdy", 0}, {"in1_rdy", 0}, {"out_vld", 0}, {"idle", 0}}, + 41, 69, {{"in0_rdy", 0}, {"in1_rdy", 0}, {"out_vld", 0}, {"idle", 1}}, expected_outputs)); - // 9. 1 cycle of data on in0 -- allows 4-stage pipeline to drain + // 9. 1 cycle of data on in0 - allows 4-stage pipeline to drain, idle + // immediately becomes 0. // 10. After 5 more cycles (on 76th cycle), pipeline drains and block becomes - // idle + // idle XLS_ASSERT_OK(SetSignalsOverCycles( 70, 70, {{"rst_n", 1}, {"in0_vld", 1}, {"in1_vld", 0}, {"out_rdy", 1}}, inputs)); @@ -3227,8 +3493,7 @@ class MultiInputWithStatePipelinedProcTestSweepFixture } }; -TEST_P(MultiInputWithStatePipelinedProcTestSweepFixture, - DISABLED_RandomStalling) { +TEST_P(MultiInputWithStatePipelinedProcTestSweepFixture, RandomStalling) { int64_t stage_count = std::get<0>(GetParam()); bool flop_inputs = std::get<1>(GetParam()); bool flop_outputs = std::get<2>(GetParam()); @@ -3371,7 +3636,7 @@ INSTANTIATE_TEST_SUITE_P( CodegenOptions::IOKind::kZeroLatencyBuffer)), MultiInputWithStatePipelinedProcTestSweepFixture::PrintToStringParamName); -TEST_F(BlockConversionTest, DISABLED_BlockWithNonMutuallyExclusiveSends) { +TEST_F(BlockConversionTest, BlockWithNonMutuallyExclusiveSends) { auto package_ptr = std::make_unique(TestName()); Package& package = *package_ptr; @@ -3403,6 +3668,10 @@ TEST_F(BlockConversionTest, DISABLED_BlockWithNonMutuallyExclusiveSends) { // Pipelined test { + XLS_ASSERT_OK_AND_ASSIGN(std::unique_ptr p, + ClonePackage(package_ptr.get())); + XLS_ASSERT_OK_AND_ASSIGN(Proc * proc, p->GetProc(TestName())); + CodegenOptions options; options.module_name(TestName()); options.flop_inputs(true).flop_outputs(true).clock_name("clk"); @@ -3416,6 +3685,10 @@ TEST_F(BlockConversionTest, DISABLED_BlockWithNonMutuallyExclusiveSends) { // Combinational test { + XLS_ASSERT_OK_AND_ASSIGN(std::unique_ptr p, + ClonePackage(package_ptr.get())); + XLS_ASSERT_OK_AND_ASSIGN(Proc * proc, p->GetProc(TestName())); + CodegenOptions options = codegen_options(); options.module_name(TestName()); options.valid_control("input_valid", "output_valid"); @@ -3639,9 +3912,12 @@ TEST_P(MultiIOWithStatePipelinedProcTestSweepFixture, DISABLED_RandomStalling) { EXPECT_GT(output0_sequence.size(), 1000); EXPECT_GT(output1_sequence.size(), 1000); - int64_t min_output_count = output0_sequence.size() > output1_sequence.size() - ? output1_sequence.size() - : output0_sequence.size(); + size_t min_input_count = + std::min(input0_sequence.size(), input1_sequence.size()); + size_t min_output_count = + std::min(output0_sequence.size(), output1_sequence.size()); + + ASSERT_GE(min_input_count, min_output_count); int64_t prior_sum = 0; @@ -3680,7 +3956,7 @@ INSTANTIATE_TEST_SUITE_P( CodegenOptions::IOKind::kZeroLatencyBuffer)), MultiIOWithStatePipelinedProcTestSweepFixture::PrintToStringParamName); -TEST_F(BlockConversionTest, DISABLED_IOSignatureFunctionBaseToPipelinedBlock) { +TEST_F(BlockConversionTest, IOSignatureFunctionBaseToPipelinedBlock) { Package package(TestName()); Type* u32 = package.GetBitsType(32); @@ -3726,7 +4002,7 @@ TEST_F(BlockConversionTest, DISABLED_IOSignatureFunctionBaseToPipelinedBlock) { XLS_VLOG_LINES(2, block->DumpIr()); } -TEST_F(BlockConversionTest, DISABLED_IOSignatureProcToCombBlock) { +TEST_F(BlockConversionTest, IOSignatureProcToCombBlock) { Package package(TestName()); Type* u32 = package.GetBitsType(32); @@ -3766,7 +4042,7 @@ TEST_F(BlockConversionTest, DISABLED_IOSignatureProcToCombBlock) { XLS_VLOG_LINES(2, block->DumpIr()); } -TEST_F(ProcConversionTestFixture, DISABLED_ProcSendDuringReset) { +TEST_F(ProcConversionTestFixture, ProcSendDuringReset) { const std::string ir_text = R"(package test chan out(bits[32], id=1, kind=streaming, ops=send_only, flow_control=ready_valid) @@ -3822,7 +4098,72 @@ proc pipelined_proc(tkn: token, st: bits[32], init={token, 1}) { IsOkAndHolds(testing::ElementsAreArray(expected_output))); } -TEST_F(ProcConversionTestFixture, DISABLED_ProcIIGreaterThanOne) { +TEST_F(ProcConversionTestFixture, SimpleProcIIGreaterThanOne) { + const std::string ir_text = R"(package test +chan in(bits[32], id=0, kind=streaming, ops=receive_only, flow_control=ready_valid) +chan out(bits[32], id=1, kind=streaming, ops=send_only, flow_control=ready_valid) + +#[initiation_interval(2)] +proc pipelined_proc(tkn: token, st: bits[32], init={token, 0}) { + send.1: token = send(tkn, st, channel=out, id=1) + min_delay.2: token = min_delay(send.1, delay=1, id=2) + receive.3: (token, bits[32]) = receive(min_delay.2, channel=in, id=3) + tuple_index.4: token = tuple_index(receive.3, index=0, id=4) + tuple_index.5: bits[32] = tuple_index(receive.3, index=1, id=5) + next_st: () = next_value(param=st, value=tuple_index.5) + next_tkn: () = next_value(param=tkn, value=tuple_index.4) +} +)"; + + XLS_ASSERT_OK_AND_ASSIGN(std::unique_ptr package, + Parser::ParsePackage(ir_text)); + + XLS_ASSERT_OK_AND_ASSIGN(Proc * proc, package->GetProc("pipelined_proc")); + + CodegenOptions options; + options.flop_inputs(false).flop_outputs(false).clock_name("clk"); + options.valid_control("input_valid", "output_valid"); + options.reset("rst", false, false, true); + options.streaming_channel_data_suffix("_data"); + options.streaming_channel_valid_suffix("_valid"); + options.streaming_channel_ready_suffix("_ready"); + options.module_name("pipelined_proc"); + + XLS_ASSERT_OK_AND_ASSIGN( + Block * block, + ConvertToBlock(proc, options, SchedulingOptions().pipeline_stages(3))); + + std::vector sources{ + ChannelSource("in_data", "in_valid", "in_ready", 1.0, block), + }; + XLS_ASSERT_OK(sources.front().SetDataSequence({10, 20, 30})); + std::vector sinks{ + ChannelSink( + "out_data", "out_valid", "out_ready", 1.0, block, + /*reset_behavior=*/ChannelSink::BehaviorDuringReset::kIgnoreValid), + }; + + std::string reset_name = options.reset()->name(); + uint64_t reset_active = options.reset()->active_low() ? 0 : 1; + uint64_t reset_inactive = options.reset()->active_low() ? 1 : 0; + std::vector> inputs( + 20, {{reset_name, reset_inactive}}); + XLS_ASSERT_OK( + SetSignalsOverCycles(0, 9, {{reset_name, reset_active}}, inputs)); + XLS_ASSERT_OK_AND_ASSIGN(BlockIOResultsAsUint64 results, + InterpretChannelizedSequentialBlockWithUint64( + block, absl::MakeSpan(sources), + absl::MakeSpan(sinks), inputs, options.reset())); + EXPECT_THAT( + sinks.at(0).GetOutputCycleSequenceAsUint64(), + IsOkAndHolds(Skipping( + 10, ElementsAre(0, std::nullopt, 10, std::nullopt, 20, std::nullopt, + 30, std::nullopt, std::nullopt, std::nullopt)))); + EXPECT_THAT(sinks.at(0).GetOutputSequenceAsUint64(), + IsOkAndHolds(ElementsAre(0, 10, 20, 30))); +} + +TEST_F(ProcConversionTestFixture, ProcIIGreaterThanOne) { const std::string ir_text = R"(package test chan in(bits[32], id=0, kind=streaming, ops=receive_only, flow_control=ready_valid) chan out(bits[32], id=1, kind=streaming, ops=send_only, flow_control=ready_valid) @@ -3888,18 +4229,11 @@ proc pipelined_proc(tkn: token, st: bits[32], init={token, 0}) { IsOkAndHolds(Skipping( 10, ElementsAre(0, std::nullopt, 10, std::nullopt, 20, std::nullopt, 30, std::nullopt, std::nullopt, std::nullopt)))); - EXPECT_THAT(sinks.at(1).GetOutputCycleSequenceAsUint64(), - IsOkAndHolds(Skipping( - 10, ElementsAre(std::nullopt, 10, std::nullopt, 20, - std::nullopt, 30, std::nullopt, std::nullopt, - std::nullopt, std::nullopt)))); EXPECT_THAT(sinks.at(0).GetOutputSequenceAsUint64(), IsOkAndHolds(ElementsAre(0, 10, 20, 30))); - EXPECT_THAT(sinks.at(1).GetOutputSequenceAsUint64(), - IsOkAndHolds(ElementsAre(10, 20, 30))); } -TEST_F(ProcConversionTestFixture, DISABLED_ProcIIGreaterThanOneRandomStalls) { +TEST_F(ProcConversionTestFixture, ProcIIGreaterThanOneRandomStalls) { const std::string ir_text = R"(package test chan in(bits[32], id=0, kind=streaming, ops=receive_only, flow_control=ready_valid) chan out(bits[32], id=1, kind=streaming, ops=send_only, flow_control=ready_valid) @@ -3974,7 +4308,7 @@ proc pipelined_proc(tkn: token, st: bits[32], init={token, 0}) { } } -TEST_F(ProcConversionTestFixture, DISABLED_SimpleProcRandomScheduler) { +TEST_F(ProcConversionTestFixture, SimpleProcRandomScheduler) { Package package(TestName()); Type* u32 = package.GetBitsType(32); XLS_ASSERT_OK_AND_ASSIGN( @@ -3989,7 +4323,7 @@ TEST_F(ProcConversionTestFixture, DISABLED_SimpleProcRandomScheduler) { TokenlessProcBuilder pb(TestName(), "tkn", &package); BValue recv = pb.Receive(in); pb.Send(out, pb.Not(pb.Not(pb.Not(pb.Not(recv))))); - XLS_ASSERT_OK_AND_ASSIGN(Proc * proc, pb.Build({})); + XLS_ASSERT_OK(pb.Build({}).status()); for (int32_t i = 0; i < 100; ++i) { int32_t seed = 100000 + 5000 * i; @@ -4003,6 +4337,10 @@ TEST_F(ProcConversionTestFixture, DISABLED_SimpleProcRandomScheduler) { options.streaming_channel_ready_suffix("_ready"); options.module_name(absl::StrFormat("pipelined_proc-%d", i)); + XLS_ASSERT_OK_AND_ASSIGN(std::unique_ptr p, + ClonePackage(&package)); + XLS_ASSERT_OK_AND_ASSIGN(Proc * proc, p->GetProc(TestName())); + XLS_ASSERT_OK_AND_ASSIGN( Block * block, ConvertToBlock(proc, options, @@ -4063,7 +4401,7 @@ TEST_F(ProcConversionTestFixture, DISABLED_SimpleProcRandomScheduler) { } } -TEST_F(ProcConversionTestFixture, DISABLED_AddRandomScheduler) { +TEST_F(ProcConversionTestFixture, AddRandomScheduler) { Package package(TestName()); Type* u32 = package.GetBitsType(32); XLS_ASSERT_OK_AND_ASSIGN( @@ -4083,7 +4421,7 @@ TEST_F(ProcConversionTestFixture, DISABLED_AddRandomScheduler) { BValue a = pb.Receive(in_a); BValue b = pb.Receive(in_b); pb.Send(out, pb.Not(pb.Not(pb.Not(pb.Not(pb.Add(a, b)))))); - XLS_ASSERT_OK_AND_ASSIGN(Proc * proc, pb.Build({})); + XLS_ASSERT_OK(pb.Build({}).status()); for (int32_t i = 0; i < 100; ++i) { int32_t seed = 100000 + 5000 * i; @@ -4097,6 +4435,10 @@ TEST_F(ProcConversionTestFixture, DISABLED_AddRandomScheduler) { options.streaming_channel_ready_suffix("_ready"); options.module_name(absl::StrFormat("pipelined_proc-%d", i)); + XLS_ASSERT_OK_AND_ASSIGN(std::unique_ptr p, + ClonePackage(&package)); + XLS_ASSERT_OK_AND_ASSIGN(Proc * proc, p->GetProc(TestName())); + XLS_ASSERT_OK_AND_ASSIGN( Block * block, ConvertToBlock(proc, options, @@ -4162,7 +4504,7 @@ TEST_F(ProcConversionTestFixture, DISABLED_AddRandomScheduler) { } } -TEST_F(ProcConversionTestFixture, DISABLED_TwoReceivesTwoSendsRandomScheduler) { +TEST_F(ProcConversionTestFixture, TwoReceivesTwoSendsRandomScheduler) { Package package(TestName()); Type* u32 = package.GetBitsType(32); XLS_ASSERT_OK_AND_ASSIGN( @@ -4185,7 +4527,7 @@ TEST_F(ProcConversionTestFixture, DISABLED_TwoReceivesTwoSendsRandomScheduler) { TokenlessProcBuilder pb(TestName(), "tkn", &package); pb.Send(out_a, pb.Not(pb.Not(pb.Not(pb.Not(pb.Receive(in_a)))))); pb.Send(out_b, pb.Not(pb.Not(pb.Not(pb.Not(pb.Receive(in_b)))))); - XLS_ASSERT_OK_AND_ASSIGN(Proc * proc, pb.Build({})); + XLS_ASSERT_OK(pb.Build({}).status()); for (int32_t i = 0; i < 100; ++i) { int32_t seed = 100000 + 5000 * i; @@ -4199,6 +4541,10 @@ TEST_F(ProcConversionTestFixture, DISABLED_TwoReceivesTwoSendsRandomScheduler) { options.streaming_channel_ready_suffix("_ready"); options.module_name(absl::StrFormat("pipelined_proc-%d", i)); + XLS_ASSERT_OK_AND_ASSIGN(std::unique_ptr p, + ClonePackage(&package)); + XLS_ASSERT_OK_AND_ASSIGN(Proc * proc, p->GetProc(TestName())); + XLS_ASSERT_OK_AND_ASSIGN( Block * block, ConvertToBlock(proc, options, @@ -4372,7 +4718,7 @@ class NonblockingReceivesProcTestSweepFixture } }; -TEST_P(NonblockingReceivesProcTestSweepFixture, DISABLED_RandomInput) { +TEST_P(NonblockingReceivesProcTestSweepFixture, RandomInput) { int64_t stage_count = std::get<0>(GetParam()); bool flop_inputs = std::get<1>(GetParam()); bool flop_outputs = std::get<2>(GetParam()); @@ -4629,14 +4975,14 @@ class ProcWithStateTest : public BlockConversionTest { } }; -TEST_F(ProcWithStateTest, DISABLED_ProcWithStateSingleCycle) { +TEST_F(ProcWithStateTest, ProcWithStateSingleCycle) { xls::SchedulingOptions scheduling_options; scheduling_options.pipeline_stages(1); TestBlockWithSchedule(scheduling_options); } -TEST_F(ProcWithStateTest, DISABLED_ProcWithStateBackedgesIn2Stages) { +TEST_F(ProcWithStateTest, ProcWithStateBackedgesIn2Stages) { xls::SchedulingOptions scheduling_options; scheduling_options.pipeline_stages(2); @@ -4647,7 +4993,7 @@ TEST_F(ProcWithStateTest, DISABLED_ProcWithStateBackedgesIn2Stages) { TestBlockWithSchedule(scheduling_options); } -TEST_F(ProcWithStateTest, DISABLED_ProcWithStateBackedgesIn3Stages) { +TEST_F(ProcWithStateTest, ProcWithStateBackedgesIn3Stages) { xls::SchedulingOptions scheduling_options; scheduling_options.pipeline_stages(3); @@ -4661,7 +5007,7 @@ TEST_F(ProcWithStateTest, DISABLED_ProcWithStateBackedgesIn3Stages) { TestBlockWithSchedule(scheduling_options); } -TEST_F(ProcWithStateTest, DISABLED_ProcWithStateBackedgesIn3StagesWithExtra) { +TEST_F(ProcWithStateTest, ProcWithStateBackedgesIn3StagesWithExtra) { xls::SchedulingOptions scheduling_options; scheduling_options.pipeline_stages(4); @@ -4685,7 +5031,7 @@ INSTANTIATE_TEST_SUITE_P( CodegenOptions::IOKind::kSkidBuffer)), NonblockingReceivesProcTestSweepFixture::PrintToStringParamName); -TEST_F(ProcConversionTestFixture, DISABLED_RecvDataFeedingSendPredicate) { +TEST_F(ProcConversionTestFixture, RecvDataFeedingSendPredicate) { Package package(TestName()); Type* u32 = package.GetBitsType(32); XLS_ASSERT_OK_AND_ASSIGN( @@ -4713,7 +5059,7 @@ TEST_F(ProcConversionTestFixture, DISABLED_RecvDataFeedingSendPredicate) { pb.SendIf(out0, lt_two_five, recv); pb.SendIf(out1, gt_one_five, recv); - XLS_ASSERT_OK_AND_ASSIGN(Proc * proc, pb.Build({})); + XLS_ASSERT_OK(pb.Build({}).status()); for (int32_t i = 0; i < 100; ++i) { int32_t seed = 100000 + 5000 * i; @@ -4727,6 +5073,10 @@ TEST_F(ProcConversionTestFixture, DISABLED_RecvDataFeedingSendPredicate) { options.streaming_channel_ready_suffix("_ready"); options.module_name(absl::StrFormat("pipelined_proc-%d", i)); + XLS_ASSERT_OK_AND_ASSIGN(std::unique_ptr p, + ClonePackage(&package)); + XLS_ASSERT_OK_AND_ASSIGN(Proc * proc, p->GetProc(TestName())); + XLS_ASSERT_OK_AND_ASSIGN( Block * block, ConvertToBlock(proc, options, @@ -4803,7 +5153,7 @@ TEST_F(ProcConversionTestFixture, DISABLED_RecvDataFeedingSendPredicate) { } } -TEST_F(ProcConversionTestFixture, DISABLED_SingleLoopbackChannel) { +TEST_F(ProcConversionTestFixture, SingleLoopbackChannel) { constexpr std::string_view ir_text = R"(package test chan loopback(bits[32], id=0, kind=streaming, ops=send_receive, flow_control=ready_valid, fifo_depth=1, register_pop_outputs=true, register_push_outputs=true) chan out(bits[32], id=1, kind=streaming, ops=send_only, flow_control=ready_valid) @@ -4847,7 +5197,7 @@ proc loopback_proc(tkn: token, st: bits[32], init={token, 1}) { // model to evaluate the block with. } -TEST_F(ProcConversionTestFixture, DISABLED_MultipleLoopbackChannel) { +TEST_F(ProcConversionTestFixture, MultipleLoopbackChannel) { constexpr std::string_view ir_text = R"(package test chan loopback0(bits[32], id=0, kind=streaming, ops=send_receive, flow_control=ready_valid, fifo_depth=1, register_push_outputs=true, register_pop_outputs=true) chan loopback1(bits[32], id=1, kind=streaming, ops=send_receive, flow_control=ready_valid, fifo_depth=1, register_push_outputs=true, register_pop_outputs=true) @@ -5129,7 +5479,7 @@ proc proc_ut(st: bits[32], init={0}) { } } -TEST_F(ProcConversionTestFixture, DISABLED_b315378547) { +TEST_F(ProcConversionTestFixture, b315378547) { const std::string ir_text = R"(package test chan out(bits[8], id=0, kind=single_value, ops=send_only) @@ -5229,7 +5579,7 @@ MATCHER_P(StateRegFoundInBlock, block, "") { return true; } -TEST_F(BlockConversionTest, DISABLED_NoDanglingPipelinePointers) { +TEST_F(BlockConversionTest, NoDanglingPipelinePointers) { constexpr std::string_view kIrText = R"( package subrosa @@ -5274,7 +5624,7 @@ top proc proc_0(param: token, param__1: bits[18], param__2: bits[3], init={token Each(Optional(StateRegFoundInBlock(block)))); } -TEST_F(ProcConversionTestFixture, DISABLED_ProcWithConditionalNextValues) { +TEST_F(ProcConversionTestFixture, ProcWithConditionalNextValues) { const std::string ir_text = R"(package test chan out(bits[32], id=1, kind=streaming, ops=send_only, flow_control=ready_valid) @@ -5683,7 +6033,7 @@ proc alternating_counter(counter0: bits[32], counter1: bits[32], index: bits[1], 5, 10, std::nullopt, 6, 11, std::nullopt, 7)))); } -TEST_F(BlockConversionTest, DISABLED_SimpleMutualExclusiveRegions) { +TEST_F(BlockConversionTest, SimpleMutualExclusiveRegions) { auto p = CreatePackage(); ProcBuilder pb(TestName(), p.get()); XLS_ASSERT_OK_AND_ASSIGN( @@ -5738,7 +6088,7 @@ TEST_F(BlockConversionTest, DISABLED_SimpleMutualExclusiveRegions) { 1, 2)); } -TEST_F(BlockConversionTest, DISABLED_NodeToStageMapSimple) { +TEST_F(BlockConversionTest, NodeToStageMapSimple) { auto p = CreatePackage(); TokenlessProcBuilder pb(TestName(), "tok", p.get()); auto a = pb.StateElement("a", UBits(0, 2)); @@ -5777,7 +6127,7 @@ TEST_F(BlockConversionTest, DISABLED_NodeToStageMapSimple) { RecordProperty("block", p->DumpIr()); } -TEST_F(BlockConversionTest, DISABLED_NodeToStageMapMulti) { +TEST_F(BlockConversionTest, NodeToStageMapMulti) { auto p = CreatePackage(); ProcBuilder pb(TestName(), p.get()); XLS_ASSERT_OK_AND_ASSIGN( @@ -5841,8 +6191,7 @@ TEST_F(BlockConversionTest, DISABLED_NodeToStageMapMulti) { .streaming_io_and_pipeline.node_to_stage_map)); } -TEST_F(BlockConversionTest, - DISABLED_SimpleMutualExclusiveAndConcurrentRegions) { +TEST_F(BlockConversionTest, SimpleMutualExclusiveAndConcurrentRegions) { auto p = CreatePackage(); ProcBuilder pb(TestName(), p.get()); XLS_ASSERT_OK_AND_ASSIGN( @@ -5895,7 +6244,7 @@ TEST_F(BlockConversionTest, context.GetMetadataForBlock(block).concurrent_stages->IsConcurrent(1, 2)); } -TEST_F(BlockConversionTest, DISABLED_SimpleConcurrentRegions) { +TEST_F(BlockConversionTest, SimpleConcurrentRegions) { auto p = CreatePackage(); ProcBuilder pb(TestName(), p.get()); XLS_ASSERT_OK_AND_ASSIGN( @@ -5947,7 +6296,7 @@ TEST_F(BlockConversionTest, DISABLED_SimpleConcurrentRegions) { context.GetMetadataForBlock(block).concurrent_stages->IsConcurrent(1, 2)); } -TEST_F(BlockConversionTest, DISABLED_MultipleConcurrentRegions) { +TEST_F(BlockConversionTest, MultipleConcurrentRegions) { auto p = CreatePackage(); ProcBuilder pb(TestName(), p.get()); XLS_ASSERT_OK_AND_ASSIGN( @@ -6023,7 +6372,7 @@ TEST_F(BlockConversionTest, DISABLED_MultipleConcurrentRegions) { context.GetMetadataForBlock(block).concurrent_stages->IsConcurrent(3, 4)); } -TEST_F(BlockConversionTest, DISABLED_CoveringRegions) { +TEST_F(BlockConversionTest, CoveringRegions) { auto p = CreatePackage(); ProcBuilder pb(TestName(), p.get()); XLS_ASSERT_OK_AND_ASSIGN( @@ -6106,7 +6455,7 @@ TEST_F(BlockConversionTest, DISABLED_CoveringRegions) { context.GetMetadataForBlock(block).concurrent_stages->IsConcurrent(3, 4)); } -TEST_F(BlockConversionTest, DISABLED_PipelineRegisterStagesKnown) { +TEST_F(BlockConversionTest, PipelineRegisterStagesKnown) { auto p = CreatePackage(); XLS_ASSERT_OK_AND_ASSIGN( Channel * x_out, p->CreateStreamingChannel("x_out", ChannelOps::kSendOnly, @@ -6157,7 +6506,7 @@ TEST_F(BlockConversionTest, DISABLED_PipelineRegisterStagesKnown) { write_at(na, 2), write_at(na, 3), write_at(na, 4))); } -TEST_F(BlockConversionTest, DISABLED_NonTopBlockNamedModuleName) { +TEST_F(BlockConversionTest, NonTopBlockNamedModuleName) { // Block conversion creates a top block with `module_name` as the name. // This tests that we get good behavior when a non-top block has the same name // as `module_name`. @@ -6186,8 +6535,11 @@ TEST_F(BlockConversionTest, DISABLED_NonTopBlockNamedModuleName) { Block * block, ConvertToBlock( p.get(), - CodegenOptions().reset("foo", false, false, false).module_name("B"), - SchedulingOptions().pipeline_stages(2))); + CodegenOptions() + .reset("foo", false, false, false) + .clock_name("clk") + .module_name("B"), + SchedulingOptions().pipeline_stages(2).schedule_all_procs(true))); EXPECT_THAT(block, m::Block("B")); EXPECT_THAT(block->nodes(), Contains(m::Literal(24))); @@ -6199,7 +6551,7 @@ TEST_F(BlockConversionTest, DISABLED_NonTopBlockNamedModuleName) { EXPECT_THAT(p->GetBlock("B__1").value()->nodes(), Contains(m::Literal(48))); } -TEST_F(ProcConversionTestFixture, DISABLED_SimpleMultiProcConversion) { +TEST_F(ProcConversionTestFixture, SimpleMultiProcConversion) { XLS_ASSERT_OK_AND_ASSIGN(std::unique_ptr p, CreateMultiProcPackage()); SchedulingOptionsFlagsProto scheduling_options; @@ -6394,7 +6746,7 @@ TEST_F(ProcConversionTestFixture, } } -TEST_F(ProcConversionTestFixture, DISABLED_SimpleFunctionWithProcsPresent) { +TEST_F(ProcConversionTestFixture, SimpleFunctionWithProcsPresent) { XLS_ASSERT_OK_AND_ASSIGN(std::unique_ptr p, CreateMultiProcPackage(/*with_functions=*/true)); XLS_ASSERT_OK_AND_ASSIGN(Function * f0, p->GetFunction("f0")); @@ -6446,8 +6798,7 @@ absl::StatusOr CreateNewStyleAccumProc(std::string_view proc_name, return pb.Build({next_accum}); } -TEST_F(ProcConversionTestFixture, - DISABLED_TrivialProcHierarchyWithProcScopedChannels) { +TEST_F(ProcConversionTestFixture, TrivialProcHierarchyWithProcScopedChannels) { // Construct a proc which instantiates one proc which accumulates its inputs. auto p = CreatePackage(); XLS_ASSERT_OK_AND_ASSIGN(Proc * leaf_proc, @@ -6469,8 +6820,9 @@ TEST_F(ProcConversionTestFixture, ProcElaboration::Elaborate(top)); XLS_ASSERT_OK(ConvertToBlock( - p.get(), CodegenOptions().reset("rst", false, false, false), - SchedulingOptions().pipeline_stages(2))); + p.get(), + CodegenOptions().reset("rst", false, false, false).clock_name("clk"), + SchedulingOptions().pipeline_stages(2).schedule_all_procs(true))); EXPECT_EQ(p->blocks().size(), 2); } diff --git a/xls/codegen_v_1_5/block_conversion_wrapper_pass.cc b/xls/codegen_v_1_5/block_conversion_wrapper_pass.cc new file mode 100644 index 0000000000..a3ce2b6054 --- /dev/null +++ b/xls/codegen_v_1_5/block_conversion_wrapper_pass.cc @@ -0,0 +1,32 @@ +// Copyright 2021 The XLS Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "xls/codegen_v_1_5/block_conversion_wrapper_pass.h" + +#include "absl/status/statusor.h" +#include "xls/codegen_v_1_5/block_conversion_pass.h" +#include "xls/ir/package.h" +#include "xls/passes/optimization_pass.h" +#include "xls/passes/pass_base.h" + +namespace xls::codegen { + +absl::StatusOr BlockConversionWrapperPass::RunInternal( + Package* package, const BlockConversionPassOptions& options, + PassResults* results) const { + return wrapped_pass_->Run(package, OptimizationPassOptions(options), results, + opt_context_); +} + +} // namespace xls::codegen diff --git a/xls/codegen_v_1_5/block_conversion_wrapper_pass.h b/xls/codegen_v_1_5/block_conversion_wrapper_pass.h new file mode 100644 index 0000000000..caa9025638 --- /dev/null +++ b/xls/codegen_v_1_5/block_conversion_wrapper_pass.h @@ -0,0 +1,74 @@ +// Copyright 2021 The XLS Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef XLS_CODEGEN_V_1_5_BLOCK_CONVERSION_WRAPPER_PASS_H_ +#define XLS_CODEGEN_V_1_5_BLOCK_CONVERSION_WRAPPER_PASS_H_ + +#include +#include + +#include "absl/status/statusor.h" +#include "absl/strings/str_format.h" +#include "absl/types/span.h" +#include "xls/codegen_v_1_5/block_conversion_pass.h" +#include "xls/ir/package.h" +#include "xls/passes/optimization_pass.h" +#include "xls/passes/pass_base.h" + +namespace xls::codegen { + +// A codegen pass wrapper which wraps a OptimizationFunctionBasePass. This is +// useful for adding an optimization or transformation pass (most passes in +// xls/passes are OptimizationFunctionBasePasses) to the codegen pipeline. The +// wrapped pass is run on the block being lowered to Verilog. +// +// Takes the OptimizationContext object at construction, since it's specific to +// optimization passes & cannot be passed via a codegen pass's Run function. +class BlockConversionWrapperPass : public BlockConversionPass { + public: + explicit BlockConversionWrapperPass( + std::unique_ptr wrapped_pass, + OptimizationContext& opt_context) + : BlockConversionPass( + absl::StrFormat("codegen_%s", wrapped_pass->short_name()), + absl::StrFormat("%s (codegen)", wrapped_pass->long_name())), + wrapped_pass_(std::move(wrapped_pass)), + opt_context_(opt_context) {} + ~BlockConversionWrapperPass() override = default; + + bool IsCompound() const override { return wrapped_pass_->IsCompound(); } + + absl::StatusOr RunInternal(Package* package, + const BlockConversionPassOptions& options, + PassResults* results) const final; + + absl::StatusOr RunNested(Package* package, + const BlockConversionPassOptions& options, + PassResults* results, + PassInvocation& invocation, + absl::Span + invariant_checkers) const override { + return wrapped_pass_->RunNested(package, OptimizationPassOptions(options), + results, opt_context_, invocation, + /*invariant_checkers=*/{}); + } + + private: + std::unique_ptr wrapped_pass_; + OptimizationContext& opt_context_; +}; + +} // namespace xls::codegen + +#endif // XLS_CODEGEN_V_1_5_BLOCK_CONVERSION_WRAPPER_PASS_H_ diff --git a/xls/codegen_v_1_5/block_conversion_wrapper_pass_test.cc b/xls/codegen_v_1_5/block_conversion_wrapper_pass_test.cc new file mode 100644 index 0000000000..45600011a9 --- /dev/null +++ b/xls/codegen_v_1_5/block_conversion_wrapper_pass_test.cc @@ -0,0 +1,76 @@ +// Copyright 2021 The XLS Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "xls/codegen_v_1_5/block_conversion_wrapper_pass.h" + +#include + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/status/statusor.h" +#include "xls/codegen_v_1_5/block_conversion_pass.h" +#include "xls/common/status/matchers.h" +#include "xls/ir/block.h" +#include "xls/ir/function_builder.h" +#include "xls/ir/ir_matcher.h" +#include "xls/ir/ir_test_base.h" +#include "xls/passes/dce_pass.h" +#include "xls/passes/optimization_pass.h" +#include "xls/passes/pass_base.h" + +namespace m = ::xls::op_matchers; + +namespace xls::codegen { +namespace { + +using ::absl_testing::IsOkAndHolds; + +class BlockConversionWrapperPassTest : public IrTestBase { + protected: + absl::StatusOr Run(Block* block) { + PassResults results; + OptimizationContext optimization_context; + return BlockConversionWrapperPass( + std::make_unique(), + optimization_context) + .Run(block->package(), BlockConversionPassOptions(), &results); + } +}; + +TEST_F(BlockConversionWrapperPassTest, WrappedDce) { + auto p = CreatePackage(); + + BlockBuilder bb(TestName(), p.get()); + XLS_ASSERT_OK(bb.block()->AddClockPort("clk")); + BValue a = bb.InputPort("a", p->GetBitsType(32)); + BValue b = bb.InputPort("b", p->GetBitsType(32)); + bb.Add(a, b); + bb.OutputPort("c", bb.Subtract(a, b)); + XLS_ASSERT_OK_AND_ASSIGN(Block * block, bb.Build()); + + // The add should be eliminated. + EXPECT_EQ(block->node_count(), 5); + EXPECT_THAT(Run(block), IsOkAndHolds(true)); + EXPECT_EQ(block->node_count(), 4); + + EXPECT_THAT(FindNode("c", block), + m::OutputPort(m::Sub(m::InputPort("a"), m::InputPort("b")))); + + // Pass should be idempotent. + EXPECT_THAT(Run(block), IsOkAndHolds(false)); + EXPECT_EQ(block->node_count(), 4); +} + +} // namespace +} // namespace xls::codegen diff --git a/xls/codegen_v_1_5/channel_to_port_io_lowering_pass.cc b/xls/codegen_v_1_5/channel_to_port_io_lowering_pass.cc index 243894a632..7c6a2f5bab 100644 --- a/xls/codegen_v_1_5/channel_to_port_io_lowering_pass.cc +++ b/xls/codegen_v_1_5/channel_to_port_io_lowering_pass.cc @@ -39,6 +39,7 @@ #include "absl/strings/str_format.h" #include "absl/types/span.h" #include "cppitertools/zip.hpp" +#include "xls/codegen/codegen_options.h" #include "xls/codegen/conversion_utils.h" #include "xls/codegen/ram_configuration.h" #include "xls/codegen_v_1_5/block_conversion_pass.h" @@ -316,9 +317,14 @@ absl::StatusOr AddPortsForSend( .ready = ready}; XLS_RETURN_IF_ERROR(block->AddChannelPortMetadata( - channel, ChannelDirection::kSend, connector.data->GetName(), - GetOptionalNodeName(connector.valid), - GetOptionalNodeName(connector.ready))); + ChannelPortMetadata{.channel_name = std::string(ChannelRefName(channel)), + .type = ChannelRefType(channel), + .direction = ChannelDirection::kSend, + .channel_kind = ChannelRefKind(channel), + .flop_kind = FlopKind::kNone, + .data_port = data->GetName(), + .valid_port = GetOptionalNodeName(valid), + .ready_port = GetOptionalNodeName(ready)})); return connector; } @@ -372,9 +378,14 @@ absl::StatusOr AddPortsForReceive( .ready = ready}; XLS_RETURN_IF_ERROR(block->AddChannelPortMetadata( - channel, ChannelDirection::kReceive, connector.data->GetName(), - GetOptionalNodeName(connector.valid), - GetOptionalNodeName(connector.ready))); + ChannelPortMetadata{.channel_name = std::string(ChannelRefName(channel)), + .type = ChannelRefType(channel), + .direction = ChannelDirection::kReceive, + .channel_kind = ChannelRefKind(channel), + .flop_kind = FlopKind::kNone, + .data_port = data->GetName(), + .valid_port = GetOptionalNodeName(valid), + .ready_port = GetOptionalNodeName(ready)})); return connector; } @@ -627,21 +638,29 @@ absl::Status ConnectReceivesToConnector( } // Connect the data & valid lines to all users (passing tokens through). - // Collect each stage's ready signal, which we'll OR together with the - // existing ready signal for the result. + // Collect each stage's ready signal, and OR them together to form the + // connector's ready signal. // In other words, we are ready to receive data from this connector if any // receive is active & ready. std::vector ready_signals; - if (connector.ready.has_value()) { - ready_signals.reserve(1 + receives.size()); - ready_signals.push_back(*connector.ReadySignal()); - } + ready_signals.reserve(receives.size()); for (const auto& [receive, stage_index] : iter::zip(receives, stage_indices)) { Stage& stage = block->stages()[stage_index]; Node* token = receive->As()->token(); + bool is_blocking = receive->As()->is_blocking(); std::optional predicate = receive->As()->predicate(); + // If needed, add identity nodes to signal that the predicate needs to be + // available at the receive's stage. (This enables pipeline register + // insertion later.) + if (predicate.has_value() && block->IsStaged(*predicate) && + *block->GetStageIndex(*predicate) != stage_index) { + XLS_ASSIGN_OR_RETURN( + predicate, block->MakeNodeInStage(stage_index, receive->loc(), + *predicate, Op::kIdentity)); + } + if (connector.ready.has_value()) { // The ready signal from this receive is: // (predicate AND outputs_ready AND outputs_valid) @@ -658,7 +677,7 @@ absl::Status ConnectReceivesToConnector( ready_signals.push_back(recv_finishing); } - if (connector.valid.has_value()) { + if (connector.valid.has_value() && is_blocking) { // This active input is valid iff the receive is inactive (!predicate) // or the valid signal is asserted. Node* recv_valid_or_inactive = *connector.valid; @@ -674,26 +693,68 @@ absl::Status ConnectReceivesToConnector( absl::MakeConstSpan({*connector.valid, recv_inactive}), Op::kOr)); } - XLS_RETURN_IF_ERROR( - ReplaceWithAnd(stage.active_inputs_valid(), recv_valid_or_inactive) - .status()); + + XLS_RETURN_IF_ERROR(ReplaceWithAnd(stage.active_inputs_valid(), + recv_valid_or_inactive, + /*combine_literals=*/false) + .status()); + } + + Node* data = connector.data; + if (options.codegen_options.gate_recvs()) { + std::vector gate_conditions; + gate_conditions.reserve(2); + if (predicate.has_value()) { + gate_conditions.push_back(*predicate); + } + if (!is_blocking && connector.valid.has_value()) { + gate_conditions.push_back(*connector.valid); + } + + Node* gate_condition = nullptr; + if (gate_conditions.size() == 1) { + gate_condition = gate_conditions.front(); + } else if (gate_conditions.size() > 1) { + XLS_ASSIGN_OR_RETURN(gate_condition, block->MakeNodeInStage( + stage_index, receive->loc(), + gate_conditions, Op::kAnd)); + } + + if (gate_condition != nullptr) { + XLS_ASSIGN_OR_RETURN(Node * zero, + block->MakeNodeInStage( + stage_index, receive->loc(), + ZeroOfType(connector.data->GetType()))); + XLS_ASSIGN_OR_RETURN(data, + block->MakeNodeInStage