diff --git a/.editorconfig b/.editorconfig index e9d27c283..80ffcbdeb 100644 --- a/.editorconfig +++ b/.editorconfig @@ -1,4 +1,4 @@ -# Copyright 2018-2021 VMware, Inc. +# Copyright 2018-2026 VMware, Inc. # SPDX-License-Identifier: Apache-2.0 # https://editorconfig.org/ diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 396a8625f..2286d4e4e 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -12,7 +12,7 @@ jobs: - uses: actions/checkout@v4 - uses: awalsh128/cache-apt-pkgs-action@latest with: - packages: libxxhash-dev libaio-dev libconfig-dev gcc clang-16 clang-format-16 + packages: libxxhash-dev libaio-dev libconfig-dev gcc clang-19 clang-format-19 - name: format-check run: ./format-check.sh @@ -78,7 +78,7 @@ jobs: - name: Maximize build space uses: easimon/maximize-build-space@master with: - root-reserve-mb: 512 + root-reserve-mb: 2048 swap-size-mb: 1 remove-dotnet: true remove-android: true @@ -88,6 +88,6 @@ jobs: - uses: actions/checkout@v4 - uses: awalsh128/cache-apt-pkgs-action@latest with: - packages: libxxhash-dev libaio-dev libconfig-dev gcc clang-16 clang-format-16 + packages: libxxhash-dev libaio-dev libconfig-dev gcc clang-19 clang-format-19 - name: make ${{ matrix.target }} - run: make ${{ matrix.target }} + run: pwd; make ${{ matrix.target }} diff --git a/Dockerfile b/Dockerfile index 0cef24733..ca9da69d7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -# Copyright 2018-2021 VMware, Inc. +# Copyright 2018-2026 VMware, Inc. # SPDX-License-Identifier: Apache-2.0 # Source for the image diff --git a/Dockerfile.build-env b/Dockerfile.build-env index 086e94136..6010b6b14 100644 --- a/Dockerfile.build-env +++ b/Dockerfile.build-env @@ -1,4 +1,4 @@ -# Copyright 2018-2021 VMware, Inc. +# Copyright 2018-2026 VMware, Inc. # SPDX-License-Identifier: Apache-2.0 # Source for the image diff --git a/Dockerfile.run-env b/Dockerfile.run-env index 21ae15ade..76929c0db 100644 --- a/Dockerfile.run-env +++ b/Dockerfile.run-env @@ -1,4 +1,4 @@ -# Copyright 2018-2021 VMware, Inc. +# Copyright 2018-2026 VMware, Inc. # SPDX-License-Identifier: Apache-2.0 # Source for the image diff --git a/Makefile b/Makefile index 0c316a4c5..216af2992 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -# Copyright 2018-2021 VMware, Inc. +# Copyright 2018-2026 VMware, Inc. # SPDX-License-Identifier: Apache-2.0 .DEFAULT_GOAL := all @@ -387,10 +387,16 @@ $(foreach unit,$(UNIT_TESTBINS),$(eval $(call unit_test_self_dependency,$(unit)) # # These will need to be fleshed out for filters, io subsystem, trunk, # etc. as we create mini unit test executables for those subsystems. -PLATFORM_SYS = $(OBJDIR)/$(SRCDIR)/$(PLATFORM_DIR)/platform.o \ +PLATFORM_SYS = $(OBJDIR)/$(SRCDIR)/$(PLATFORM_DIR)/platform_assert.o \ + $(OBJDIR)/$(SRCDIR)/$(PLATFORM_DIR)/platform_buffer.o \ + $(OBJDIR)/$(SRCDIR)/$(PLATFORM_DIR)/platform_condvar.o \ + $(OBJDIR)/$(SRCDIR)/$(PLATFORM_DIR)/platform_heap.o \ + $(OBJDIR)/$(SRCDIR)/$(PLATFORM_DIR)/platform_log.o \ + $(OBJDIR)/$(SRCDIR)/$(PLATFORM_DIR)/platform_mutex.o \ + $(OBJDIR)/$(SRCDIR)/$(PLATFORM_DIR)/platform_threads.o \ $(OBJDIR)/$(SRCDIR)/$(PLATFORM_DIR)/shmem.o - -PLATFORM_IO_SYS = $(OBJDIR)/$(SRCDIR)/$(PLATFORM_DIR)/laio.o +PLATFORM_IO_SYS = $(OBJDIR)/$(SRCDIR)/$(PLATFORM_DIR)/platform_io.o \ + $(OBJDIR)/$(SRCDIR)/$(PLATFORM_DIR)/laio.o UTIL_SYS = $(OBJDIR)/$(SRCDIR)/util.o $(PLATFORM_SYS) diff --git a/NOTICE b/NOTICE index 79d82ee82..d854c3600 100644 --- a/NOTICE +++ b/NOTICE @@ -1,4 +1,4 @@ -Copyright 2021 VMware, Inc. +Copyright 2021-2026 VMware, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/ci/apply-pipeline.sh b/ci/apply-pipeline.sh index 2d405399b..5b73252a3 100755 --- a/ci/apply-pipeline.sh +++ b/ci/apply-pipeline.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2018-2021 VMware, Inc. +# Copyright 2018-2026 VMware, Inc. # SPDX-License-Identifier: Apache-2.0 # Reconfigure the Concourse CI pipeline for this project. diff --git a/ci/jobs.lib.yml b/ci/jobs.lib.yml index 79c49bba2..e0799e8fb 100644 --- a/ci/jobs.lib.yml +++ b/ci/jobs.lib.yml @@ -1,4 +1,4 @@ -#! Copyright 2018-2021 VMware, Inc. +#! Copyright 2018-2026 VMware, Inc. #! SPDX-License-Identifier: Apache-2.0 #@ load("@ytt:template", "template") diff --git a/ci/pipeline.yml b/ci/pipeline.yml index a4f56efe9..13219a2bf 100644 --- a/ci/pipeline.yml +++ b/ci/pipeline.yml @@ -1,4 +1,4 @@ -#! Copyright 2018-2021 VMware, Inc. +#! Copyright 2018-2026 VMware, Inc. #! SPDX-License-Identifier: Apache-2.0 #! A ytt template for our Concourse CI pipeline diff --git a/ci/resources.lib.yml b/ci/resources.lib.yml index 81a4d5f7b..80e53228e 100644 --- a/ci/resources.lib.yml +++ b/ci/resources.lib.yml @@ -1,4 +1,4 @@ -#! Copyright 2018-2021 VMware, Inc. +#! Copyright 2018-2026 VMware, Inc. #! SPDX-License-Identifier: Apache-2.0 #@ load("@ytt:template", "template") diff --git a/ci/sequences.lib.yml b/ci/sequences.lib.yml index f269bd977..99127ad4e 100644 --- a/ci/sequences.lib.yml +++ b/ci/sequences.lib.yml @@ -1,4 +1,4 @@ -#! Copyright 2018-2021 VMware, Inc. +#! Copyright 2018-2026 VMware, Inc. #! SPDX-License-Identifier: Apache-2.0 #@ load("steps.lib.yml", diff --git a/ci/steps.lib.yml b/ci/steps.lib.yml index 2c52a3cda..a3d34385a 100644 --- a/ci/steps.lib.yml +++ b/ci/steps.lib.yml @@ -1,4 +1,4 @@ -#! Copyright 2018-2021 VMware, Inc. +#! Copyright 2018-2026 VMware, Inc. #! SPDX-License-Identifier: Apache-2.0 --- diff --git a/ci/tasks/pr-check-shell-scripts.sh b/ci/tasks/pr-check-shell-scripts.sh index f90b3ceac..03fac877e 100755 --- a/ci/tasks/pr-check-shell-scripts.sh +++ b/ci/tasks/pr-check-shell-scripts.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2018-2021 VMware, Inc. +# Copyright 2018-2026 VMware, Inc. # SPDX-License-Identifier: Apache-2.0 # Lints and formats any shell scripts changed by a PR diff --git a/default.cfg b/default.cfg index 9e95c6f0d..5bba89f5c 100644 --- a/default.cfg +++ b/default.cfg @@ -1,4 +1,4 @@ -# Copyright 2018-2021 VMware, Inc. +# Copyright 2018-2026 VMware, Inc. # SPDX-License-Identifier: Apache-2.0 ############################## diff --git a/docs/build.md b/docs/build.md index 75532bf62..80a1161ac 100644 --- a/docs/build.md +++ b/docs/build.md @@ -7,9 +7,9 @@ To integrate SplinterDB into another application, see [Usage](usage.md). Builds are known to work on Ubuntu using recent versions of GCC and Clang. ### Tooling -In CI, we test against GCC 13 and Clang 16. +In CI, we test against GCC 13 and Clang 19. -We use `clang-format-16` for code formatting. +We use `clang-format-19` for code formatting. ### Full build Here are the steps to do a full-build of the library, run smoke tests, and to install the shared libraries: diff --git a/docs/site/content/docs/build.md b/docs/site/content/docs/build.md index 75532bf62..80a1161ac 100644 --- a/docs/site/content/docs/build.md +++ b/docs/site/content/docs/build.md @@ -7,9 +7,9 @@ To integrate SplinterDB into another application, see [Usage](usage.md). Builds are known to work on Ubuntu using recent versions of GCC and Clang. ### Tooling -In CI, we test against GCC 13 and Clang 16. +In CI, we test against GCC 13 and Clang 19. -We use `clang-format-16` for code formatting. +We use `clang-format-19` for code formatting. ### Full build Here are the steps to do a full-build of the library, run smoke tests, and to install the shared libraries: diff --git a/docs/site/content/docs/v0.0.1/build.md b/docs/site/content/docs/v0.0.1/build.md index b862d61b2..98ae4de6c 100644 --- a/docs/site/content/docs/v0.0.1/build.md +++ b/docs/site/content/docs/v0.0.1/build.md @@ -7,9 +7,9 @@ To integrate SplinterDB into another application, see [Usage](usage.md). Builds are known to work on Ubuntu using recent versions of GCC and Clang. ### Tooling -In CI, we test against GCC 13 and Clang 16. +In CI, we test against GCC 13 and Clang 19. -We use `clang-format-16` for code formatting. +We use `clang-format-19` for code formatting. ### Full build Here are the steps to do a full-build of the library, run smoke tests, and to install the shared libraries: diff --git a/examples/splinterdb_custom_ipv4_addr_sortcmp_example.c b/examples/splinterdb_custom_ipv4_addr_sortcmp_example.c index dd613c02c..c534f5141 100644 --- a/examples/splinterdb_custom_ipv4_addr_sortcmp_example.c +++ b/examples/splinterdb_custom_ipv4_addr_sortcmp_example.c @@ -1,4 +1,4 @@ -// Copyright 2022 VMware, Inc. +// Copyright 2022-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* diff --git a/examples/splinterdb_intro_example.c b/examples/splinterdb_intro_example.c index 7d8232be0..18e6f0cf3 100644 --- a/examples/splinterdb_intro_example.c +++ b/examples/splinterdb_intro_example.c @@ -1,4 +1,4 @@ -// Copyright 2022 VMware, Inc. +// Copyright 2022-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* * SplinterDB "Hello World" example program. Demonstrate use of: diff --git a/examples/splinterdb_iterators_example.c b/examples/splinterdb_iterators_example.c index 89437e49f..cfb767ec1 100644 --- a/examples/splinterdb_iterators_example.c +++ b/examples/splinterdb_iterators_example.c @@ -1,4 +1,4 @@ -// Copyright 2022 VMware, Inc. +// Copyright 2022-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* diff --git a/examples/splinterdb_wide_values_example.c b/examples/splinterdb_wide_values_example.c index 8f7cfda25..366d9d34d 100644 --- a/examples/splinterdb_wide_values_example.c +++ b/examples/splinterdb_wide_values_example.c @@ -1,4 +1,4 @@ -// Copyright 2022 VMware, Inc. +// Copyright 2022-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* diff --git a/format-check.sh b/format-check.sh index 386aff469..0fe9368e5 100755 --- a/format-check.sh +++ b/format-check.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2018-2022 VMware, Inc. +# Copyright 2018-2026 VMware, Inc. # SPDX-License-Identifier: Apache-2.0 # Checks that C source files follow our formatting conventions @@ -9,7 +9,7 @@ set -eu -o pipefail # different tool versions yield different results # so we standardize on this version -TOOL="clang-format-16" +TOOL="clang-format-19" # Check if TOOL exists if ! command -v "$TOOL" &> /dev/null; then diff --git a/include/splinterdb/data.h b/include/splinterdb/data.h index 2ad7f118e..aeaaa6f80 100644 --- a/include/splinterdb/data.h +++ b/include/splinterdb/data.h @@ -1,4 +1,4 @@ -// Copyright 2018-2021 VMware, Inc. +// Copyright 2018-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* diff --git a/include/splinterdb/default_data_config.h b/include/splinterdb/default_data_config.h index 1621ea979..4be9e2e65 100644 --- a/include/splinterdb/default_data_config.h +++ b/include/splinterdb/default_data_config.h @@ -1,4 +1,4 @@ -// Copyright 2018-2021 VMware, Inc. +// Copyright 2018-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 // A default data_config suitable for simple key/value applications @@ -7,14 +7,11 @@ // This data_config does not support blind mutation operations, except // plain overwrites of values. -#ifndef _SPLINTERDB_DEFAULT_DATA_CONFIG_H_ -#define _SPLINTERDB_DEFAULT_DATA_CONFIG_H_ +#pragma once -#include "splinterdb/data.h" +#include "data.h" void default_data_config_init(const uint64 max_key_size, // IN data_config *out_cfg // OUT ); - -#endif // _SPLINTERDB_DEFAULT_DATA_CONFIG_H_ diff --git a/include/splinterdb/platform_linux/public_platform.h b/include/splinterdb/platform_linux/public_platform.h index ec97447b8..e05798604 100644 --- a/include/splinterdb/platform_linux/public_platform.h +++ b/include/splinterdb/platform_linux/public_platform.h @@ -1,4 +1,4 @@ -// Copyright 2018-2021 VMware, Inc. +// Copyright 2018-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* @@ -43,6 +43,8 @@ typedef uint64_t uint64; typedef uint64 timestamp; typedef uint64 threadid; +typedef int32 bool32; + #include static_assert(sizeof(int8) == 1, "incorrect type"); static_assert(sizeof(uint8) == 1, "incorrect type"); @@ -74,3 +76,24 @@ typedef FILE platform_log_handle; void platform_set_log_streams(platform_log_handle *info_stream, platform_log_handle *error_stream); + +// Register the current thread so that it can be used with splinterdb. +// +// Any thread that uses a splinterdb must first be registered with it. +// +// The only exception is the initial thread which called create or open, +// as that thread is implicitly registered. Re-registering it is an error. +// +// A thread should not be registered more than once; that is an error. +// +// Note: There is currently a limit of MAX_THREADS registered at a given time +// +// Returns 0 on success, -1 on error. +int +platform_register_thread(void); + +// Deregister the current thread. +// +// Call this function before exiting a registered thread. +void +platform_deregister_thread(void); diff --git a/include/splinterdb/public_platform.h b/include/splinterdb/public_platform.h index e71a2d8ee..23b121269 100644 --- a/include/splinterdb/public_platform.h +++ b/include/splinterdb/public_platform.h @@ -1,4 +1,4 @@ -// Copyright 2018-2021 VMware, Inc. +// Copyright 2018-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* diff --git a/include/splinterdb/public_util.h b/include/splinterdb/public_util.h index e20406a69..eba658912 100644 --- a/include/splinterdb/public_util.h +++ b/include/splinterdb/public_util.h @@ -1,4 +1,4 @@ -// Copyright 2018-2021 VMware, Inc. +// Copyright 2018-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 #pragma once @@ -25,7 +25,7 @@ typedef struct slice { } slice; #define NULL_SLICE ((slice){.length = 0, .data = NULL}) -#define INVALID_SLICE ((slice){.length = (uint64)-1, .data = NULL}) +#define INVALID_SLICE ((slice){.length = (uint64) - 1, .data = NULL}) static inline _Bool slice_is_null(const slice b) diff --git a/include/splinterdb/splinterdb.h b/include/splinterdb/splinterdb.h index e861a1ac4..4060bae9b 100644 --- a/include/splinterdb/splinterdb.h +++ b/include/splinterdb/splinterdb.h @@ -1,4 +1,4 @@ -// Copyright 2018-2021 VMware, Inc. +// Copyright 2018-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* @@ -8,12 +8,15 @@ * * A data_config must be provided at the time of create/open. * See default_data_config.h for a basic reference implementation. + * + * Each thread must call splinterdb_register_thread() before making any calls to + * SplinterDB. Each thread must call splinterdb_deregister_thread() before + * exiting. */ -#ifndef _SPLINTERDB_H_ -#define _SPLINTERDB_H_ +#pragma once -#include "splinterdb/data.h" +#include "data.h" // Get a version string for this build of SplinterDB // Currently a git tag @@ -177,30 +180,6 @@ splinterdb_open(const splinterdb_config *cfg, splinterdb **kvs); void splinterdb_close(splinterdb **kvs); -// Register the current thread so that it can be used with splinterdb. -// This causes scratch space to be allocated for the thread. -// -// Any thread that uses a splinterdb must first be registered with it. -// -// The only exception is the initial thread which called create or open, -// as that thread is implicitly registered. Re-registering it will leak memory. -// -// A thread should not be registered more than once; that would leak memory. -// -// splinterdb_close will use scratch space, so the thread that calls it must -// have been registered (or implicitly registered by being the initial thread). -// -// Note: There is currently a limit of MAX_THREADS registered at a given time -void -splinterdb_register_thread(splinterdb *kvs); - -// Deregister the current thread and free its scratch space. -// -// Call this function before exiting a registered thread. -// Otherwise, you'll leak memory. -void -splinterdb_deregister_thread(splinterdb *kvs); - // Insert a key and value. // Relies on data_config->encode_message int @@ -420,5 +399,3 @@ splinterdb_stats_print_lookup(const splinterdb *kvs); void splinterdb_stats_reset(splinterdb *kvs); - -#endif // _SPLINTERDB_H_ diff --git a/scripts/compute_amp.sh b/scripts/compute_amp.sh index b468c460b..ca6d69fab 100755 --- a/scripts/compute_amp.sh +++ b/scripts/compute_amp.sh @@ -1,6 +1,6 @@ #!/bin/bash -x -# Copyright 2018-2021 VMware, Inc. +# Copyright 2018-2026 VMware, Inc. # SPDX-License-Identifier: Apache-2.0 dir=$1 diff --git a/scripts/config.sh b/scripts/config.sh index 1392549a2..4921dc344 100755 --- a/scripts/config.sh +++ b/scripts/config.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2018-2021 VMware, Inc. +# Copyright 2018-2026 VMware, Inc. # SPDX-License-Identifier: Apache-2.0 cat "$@" \ diff --git a/scripts/config_reader.py b/scripts/config_reader.py index b5f1aec45..1fc7df979 100755 --- a/scripts/config_reader.py +++ b/scripts/config_reader.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2018-2021 VMware, Inc. +# Copyright 2018-2026 VMware, Inc. # SPDX-License-Identifier: Apache-2.0 diff --git a/scripts/copywriter.sh b/scripts/copywriter.sh index 448b413af..fe803b474 100755 --- a/scripts/copywriter.sh +++ b/scripts/copywriter.sh @@ -1,10 +1,10 @@ #!/bin/bash -# Copyright 2018-2021 VMware, Inc. +# Copyright 2018-2026 VMware, Inc. # SPDX-License-Identifier: Apache-2.0 -# Copyright 2018-2021 VMware, Inc., Microsoft Inc., Carnegie Mellon University, ETH Zurich, and University of Washington +# Copyright 2018-2026 VMware, Inc., Microsoft Inc., Carnegie Mellon University, ETH Zurich, and University of Washington # SPDX-License-Identifier: BSD-2-Clause DRY_RUN= @@ -14,7 +14,7 @@ if [ "$1" == "-d" ]; then fi OLD_COPYRIGHT_NOTICE="" -COPYRIGHT_NOTICE="Copyright 2018-2021 VMware, Inc." +COPYRIGHT_NOTICE="Copyright 2018-2026 VMware, Inc." COPYING_PERMISSION_STATEMENT="SPDX-License-Identifier: Apache-2.0" function starts_with_bang_line() { diff --git a/scripts/list_test.py b/scripts/list_test.py index a62d3ce65..548111bfa 100755 --- a/scripts/list_test.py +++ b/scripts/list_test.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2018-2021 VMware, Inc. +# Copyright 2018-2026 VMware, Inc. # SPDX-License-Identifier: Apache-2.0 diff --git a/scripts/load_splinter.sh b/scripts/load_splinter.sh index 595b9580a..d38af8013 100755 --- a/scripts/load_splinter.sh +++ b/scripts/load_splinter.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2018-2021 VMware, Inc. +# Copyright 2018-2026 VMware, Inc. # SPDX-License-Identifier: Apache-2.0 # - Need to run this script as sudo for cgroup stuff diff --git a/scripts/load_splinter_multi.sh b/scripts/load_splinter_multi.sh index 799c9220f..6fdf1b3ab 100755 --- a/scripts/load_splinter_multi.sh +++ b/scripts/load_splinter_multi.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2018-2021 VMware, Inc. +# Copyright 2018-2026 VMware, Inc. # SPDX-License-Identifier: Apache-2.0 diff --git a/scripts/multi_seq_splinter.sh b/scripts/multi_seq_splinter.sh index 852bc06ca..e280b16b6 100755 --- a/scripts/multi_seq_splinter.sh +++ b/scripts/multi_seq_splinter.sh @@ -1,6 +1,6 @@ #!/bin/bash -x -# Copyright 2018-2021 VMware, Inc. +# Copyright 2018-2026 VMware, Inc. # SPDX-License-Identifier: Apache-2.0 # - Need to run this script as sudo for cgroup stuff diff --git a/scripts/read_splinter.sh b/scripts/read_splinter.sh index 5de60ebab..7541a76cb 100755 --- a/scripts/read_splinter.sh +++ b/scripts/read_splinter.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2018-2021 VMware, Inc. +# Copyright 2018-2026 VMware, Inc. # SPDX-License-Identifier: Apache-2.0 # - Need to run this script as sudo for cgroup stuff diff --git a/scripts/read_splinter_multi.sh b/scripts/read_splinter_multi.sh index bfa8a94d1..093a865c7 100755 --- a/scripts/read_splinter_multi.sh +++ b/scripts/read_splinter_multi.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2018-2021 VMware, Inc. +# Copyright 2018-2026 VMware, Inc. # SPDX-License-Identifier: Apache-2.0 diff --git a/scripts/run_test.sh b/scripts/run_test.sh index 324b1020d..051641022 100755 --- a/scripts/run_test.sh +++ b/scripts/run_test.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2018-2021 VMware, Inc. +# Copyright 2018-2026 VMware, Inc. # SPDX-License-Identifier: Apache-2.0 diff --git a/scripts/scan_splinter.sh b/scripts/scan_splinter.sh index f3b3334ea..b600ab5de 100755 --- a/scripts/scan_splinter.sh +++ b/scripts/scan_splinter.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2018-2021 VMware, Inc. +# Copyright 2018-2026 VMware, Inc. # SPDX-License-Identifier: Apache-2.0 # - Need to run this script as sudo for cgroup stuff diff --git a/scripts/scan_splinter_multi.sh b/scripts/scan_splinter_multi.sh index f3c4ae177..3e995965f 100755 --- a/scripts/scan_splinter_multi.sh +++ b/scripts/scan_splinter_multi.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2018-2021 VMware, Inc. +# Copyright 2018-2026 VMware, Inc. # SPDX-License-Identifier: Apache-2.0 diff --git a/scripts/seq_splinter.sh b/scripts/seq_splinter.sh index 4f219eeae..33cc70c87 100755 --- a/scripts/seq_splinter.sh +++ b/scripts/seq_splinter.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2018-2021 VMware, Inc. +# Copyright 2018-2026 VMware, Inc. # SPDX-License-Identifier: Apache-2.0 # - Need to run this script as sudo for cgroup stuff diff --git a/scripts/space_splinter.sh b/scripts/space_splinter.sh index a79f106fc..d0aa062f2 100755 --- a/scripts/space_splinter.sh +++ b/scripts/space_splinter.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2018-2021 VMware, Inc. +# Copyright 2018-2026 VMware, Inc. # SPDX-License-Identifier: Apache-2.0 # - Need to run this script as sudo for cgroup stuff diff --git a/scripts/space_splinter_multi.sh b/scripts/space_splinter_multi.sh index cfab6fabc..e0233c92f 100755 --- a/scripts/space_splinter_multi.sh +++ b/scripts/space_splinter_multi.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2018-2021 VMware, Inc. +# Copyright 2018-2026 VMware, Inc. # SPDX-License-Identifier: Apache-2.0 diff --git a/scripts/splinter_test_common b/scripts/splinter_test_common index 62b60d1bd..b44cea6ac 100644 --- a/scripts/splinter_test_common +++ b/scripts/splinter_test_common @@ -1,4 +1,4 @@ -# Copyright 2018-2021 VMware, Inc. +# Copyright 2018-2026 VMware, Inc. # SPDX-License-Identifier: Apache-2.0 # Common splinter test functions diff --git a/scripts/test_runner.sh b/scripts/test_runner.sh index 7424f8aaa..6fd37f3ca 100755 --- a/scripts/test_runner.sh +++ b/scripts/test_runner.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2018-2021 VMware, Inc. +# Copyright 2018-2026 VMware, Inc. # SPDX-License-Identifier: Apache-2.0 diff --git a/src/PackedArray.c b/src/PackedArray.c index 2621fb43d..1654d9651 100644 --- a/src/PackedArray.c +++ b/src/PackedArray.c @@ -6,12 +6,13 @@ // see README.md for usage instructions. // (‑●‑●)> released under the WTFPL v2 license, by Gregory Pakosz (@gpakosz) -#include "platform.h" #ifndef PACKEDARRAY_SELF #define PACKEDARRAY_SELF "PackedArray.c" #endif +#include "platform_assert.h" + #ifdef PACKEDARRAY_IMPL #ifndef PACKEDARRAY_JOIN diff --git a/src/PackedArray.h b/src/PackedArray.h index 8a99176f1..f452512c9 100644 --- a/src/PackedArray.h +++ b/src/PackedArray.h @@ -4,8 +4,7 @@ #ifndef PACKEDARRAY_H #define PACKEDARRAY_H -#include "platform.h" -#include "poison.h" +#include "splinterdb/public_platform.h" /* * ----------------------------------------------------------------------------- diff --git a/src/allocator.c b/src/allocator.c index 6260c0200..fa2aa37f8 100644 --- a/src/allocator.c +++ b/src/allocator.c @@ -1,4 +1,4 @@ -// Copyright 2018-2021 VMware, Inc. +// Copyright 2018-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* diff --git a/src/allocator.h b/src/allocator.h index 7ec1840f4..97a71e47a 100644 --- a/src/allocator.h +++ b/src/allocator.h @@ -1,4 +1,4 @@ -// Copyright 2018-2021 VMware, Inc. +// Copyright 2018-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* @@ -9,9 +9,8 @@ #pragma once -#include "platform.h" - -#include "io.h" +#include "platform_log.h" +#include "platform_io.h" typedef uint64 allocator_root_id; #define INVALID_ALLOCATOR_ROOT_ID (0) diff --git a/src/batch_rwlock.c b/src/batch_rwlock.c new file mode 100644 index 000000000..75249b959 --- /dev/null +++ b/src/batch_rwlock.c @@ -0,0 +1,161 @@ +// Copyright 2018-2026 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +#include "batch_rwlock.h" +#include "platform_sleep.h" +#include "poison.h" + +/* + *----------------------------------------------------------------------------- + * Batch Read/Write Locks + * + * These are generic distributed reader/writer locks with an intermediate claim + * state. These offer similar semantics to the locks in the clockcache, but in + * these locks read locks cannot be held with write locks. + * + * These locks are allocated in batches of PLATFORM_CACHELINE_SIZE / 2, so that + * the write lock and claim bits, as well as the distributed read counters, can + * be colocated across the batch. + *----------------------------------------------------------------------------- + */ + +void +batch_rwlock_init(batch_rwlock *lock) +{ + ZERO_CONTENTS(lock); +} + +void +batch_rwlock_deinit(batch_rwlock *lock) +{ + ZERO_CONTENTS(lock); +} + +/* + *----------------------------------------------------------------------------- + * lock/unlock + * + * Drains and blocks all gets (read locks) + * + * Caller must hold a claim + * Caller cannot hold a get (read lock) + *----------------------------------------------------------------------------- + */ + +void +batch_rwlock_lock(batch_rwlock *lock, uint64 lock_idx) +{ + platform_assert(lock->write_lock[lock_idx].claim); + debug_only uint8 was_locked = + __sync_lock_test_and_set(&lock->write_lock[lock_idx].lock, 1); + debug_assert(!was_locked, + "batch_rwlock_lock: Attempt to lock a locked page.\n"); + + uint64 wait = 1; + for (uint64 i = 0; i < MAX_THREADS; i++) { + while (lock->read_counter[i][lock_idx] != 0) { + platform_sleep_ns(wait); + wait = wait > 2048 ? wait : 2 * wait; + } + wait = 1; + } +} + +void +batch_rwlock_unlock(batch_rwlock *lock, uint64 lock_idx) +{ + __sync_lock_release(&lock->write_lock[lock_idx].lock); +} + +void +batch_rwlock_full_unlock(batch_rwlock *lock, uint64 lock_idx) +{ + batch_rwlock_unlock(lock, lock_idx); + batch_rwlock_unclaim(lock, lock_idx); + batch_rwlock_unget(lock, lock_idx); +} + +/* + *----------------------------------------------------------------------------- + * try_claim/claim/unlock + * + * A claim blocks all other claimants (and therefore all other writelocks, + * because writelocks are required to hold a claim during the writelock). + * + * Must hold a get (read lock) + * try_claim returns whether the claim succeeded + *----------------------------------------------------------------------------- + */ + +bool32 +batch_rwlock_try_claim(batch_rwlock *lock, uint64 lock_idx) +{ + threadid tid = platform_get_tid(); + debug_assert(lock->read_counter[tid][lock_idx]); + if (__sync_lock_test_and_set(&lock->write_lock[lock_idx].claim, 1)) { + return FALSE; + } + debug_only uint8 old_counter = + __sync_fetch_and_sub(&lock->read_counter[tid][lock_idx], 1); + debug_assert(0 < old_counter); + return TRUE; +} + +void +batch_rwlock_claim_loop(batch_rwlock *lock, uint64 lock_idx) +{ + uint64 wait = 1; + while (!batch_rwlock_try_claim(lock, lock_idx)) { + batch_rwlock_unget(lock, lock_idx); + platform_sleep_ns(wait); + wait = wait > 2048 ? wait : 2 * wait; + batch_rwlock_get(lock, lock_idx); + } +} + +void +batch_rwlock_unclaim(batch_rwlock *lock, uint64 lock_idx) +{ + threadid tid = platform_get_tid(); + __sync_fetch_and_add(&lock->read_counter[tid][lock_idx], 1); + __sync_lock_release(&lock->write_lock[lock_idx].claim); +} + +/* + *----------------------------------------------------------------------------- + * get/unget + * + * Acquire a read lock + *----------------------------------------------------------------------------- + */ + +void +batch_rwlock_get(batch_rwlock *lock, uint64 lock_idx) +{ + threadid tid = platform_get_tid(); + while (1) { + uint64 wait = 1; + while (lock->write_lock[lock_idx].lock) { + platform_sleep_ns(wait); + wait = wait > 2048 ? wait : 2 * wait; + } + debug_only uint8 old_counter = + __sync_fetch_and_add(&lock->read_counter[tid][lock_idx], 1); + debug_assert(old_counter == 0); + if (!lock->write_lock[lock_idx].lock) { + return; + } + old_counter = __sync_fetch_and_sub(&lock->read_counter[tid][lock_idx], 1); + debug_assert(old_counter == 1); + } + platform_assert(0); +} + +void +batch_rwlock_unget(batch_rwlock *lock, uint64 lock_idx) +{ + threadid tid = platform_get_tid(); + debug_only uint8 old_counter = + __sync_fetch_and_sub(&lock->read_counter[tid][lock_idx], 1); + debug_assert(old_counter == 1); +} diff --git a/src/batch_rwlock.h b/src/batch_rwlock.h new file mode 100644 index 000000000..fa22bc9fc --- /dev/null +++ b/src/batch_rwlock.h @@ -0,0 +1,93 @@ +// Copyright 2018-2026 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "splinterdb/platform_linux/public_platform.h" +#include "platform_threads.h" + +/* + *----------------------------------------------------------------------------- + * Batch Read/Write Locks + * + * These are generic distributed reader/writer locks with an intermediate claim + * state. These offer similar semantics to the locks in the clockcache, but in + * these locks read locks cannot be held with write locks. + * + * These locks are allocated in batches of PLATFORM_CACHELINE_SIZE / 2, so that + * the write lock and claim bits, as well as the distributed read counters, can + * be colocated across the batch. + *----------------------------------------------------------------------------- + */ + +// Distributed Batch RW Lock +typedef struct { + volatile uint8 lock; + volatile uint8 claim; +} batch_claimlock; + +typedef struct { + batch_claimlock write_lock[PLATFORM_CACHELINE_SIZE / 2]; + volatile uint8 read_counter[MAX_THREADS][PLATFORM_CACHELINE_SIZE / 2]; +} PLATFORM_CACHELINE_ALIGNED batch_rwlock; + +_Static_assert(sizeof(batch_rwlock) + == PLATFORM_CACHELINE_SIZE * (MAX_THREADS / 2 + 1), + "Missized batch_rwlock\n"); + + +/* + * The state machine for a thread interacting with a batch_rwlock is: + * + * get claim lock + * unlocked <-------> read-locked <----------> claimed <--------> write-locked + * unget unclaim unlock + * + * Note that try_claim() may fail, in which case the state of the lock + * is unchanged, i.e. the caller still holds a read lock. + */ + + +void +batch_rwlock_init(batch_rwlock *lock); + +void +batch_rwlock_deinit(batch_rwlock *lock); + +/* no lock -> shared lock */ +void +batch_rwlock_get(batch_rwlock *lock, uint64 lock_idx); + +/* shared lock -> no lock */ +void +batch_rwlock_unget(batch_rwlock *lock, uint64 lock_idx); + +/* + * shared-lock -> claim (may fail) + * + * Callers still hold a shared lock after a failed claim attempt. + * Callers _must_ release their shared lock after a failed claim attempt. + */ +bool32 +batch_rwlock_try_claim(batch_rwlock *lock, uint64 lock_idx); + +/* shared-lock -> claim, BUT(!) may temporarily release the shared-lock in the + * process. */ +void +batch_rwlock_claim_loop(batch_rwlock *lock, uint64 lock_idx); + +/* claim -> shared lock */ +void +batch_rwlock_unclaim(batch_rwlock *lock, uint64 lock_idx); + +/* claim -> exclusive lock */ +void +batch_rwlock_lock(batch_rwlock *lock, uint64 lock_idx); + +/* exclusive lock -> claim */ +void +batch_rwlock_unlock(batch_rwlock *lock, uint64 lock_idx); + +/* exclusive-lock -> unlocked */ +void +batch_rwlock_full_unlock(batch_rwlock *lock, uint64 lock_idx); diff --git a/src/btree.c b/src/btree.c index 83ff8817a..76a0ad0d7 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1,9 +1,9 @@ -// Copyright 2018-2021 VMware, Inc. All rights reserved. -- VMware Confidential +// Copyright 2018-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 #include "btree_private.h" +#include "platform_sleep.h" #include "poison.h" - /* * ***************************************************************** * Structure of a BTree node: Disk-resident structure: @@ -1211,13 +1211,8 @@ btree_create(cache *cc, cache_unget(cc, root_page); // set up the mini allocator - mini_init(mini, - cc, - cfg->data_cfg, - root.addr + btree_page_size(cfg), - 0, - BTREE_MAX_HEIGHT, - type); + mini_init( + mini, cc, root.addr + btree_page_size(cfg), 0, BTREE_MAX_HEIGHT, type); return root.addr; } @@ -2123,7 +2118,7 @@ btree_lookup_node_async(btree_lookup_async_state *state, uint64 depth) key_is_positive_infinity(state->target) ? btree_num_entries(state->node.hdr) - 1 : btree_find_pivot( - state->cfg, state->node.hdr, state->target, &state->found); + state->cfg, state->node.hdr, state->target, &state->found); if (child_idx < 0) { child_idx = 0; } diff --git a/src/btree.h b/src/btree.h index c8206ee75..707fa69cd 100644 --- a/src/btree.h +++ b/src/btree.h @@ -1,4 +1,4 @@ -// Copyright 2021 VMware, Inc. +// Copyright 2021-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* @@ -9,6 +9,8 @@ #pragma once +#include "platform_hash.h" +#include "platform_typed_alloc.h" #include "async.h" #include "mini_allocator.h" #include "iterator.h" diff --git a/src/btree_private.h b/src/btree_private.h index 343fd826f..99f6fb908 100644 --- a/src/btree_private.h +++ b/src/btree_private.h @@ -1,4 +1,4 @@ -// Copyright 2021 VMware, Inc. +// Copyright 2021-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* @@ -10,7 +10,6 @@ */ #pragma once -#include "splinterdb/public_platform.h" #include "splinterdb/data.h" #include "util.h" #include "btree.h" diff --git a/src/cache.h b/src/cache.h index bf3a2b8f5..a8be012ff 100644 --- a/src/cache.h +++ b/src/cache.h @@ -1,4 +1,4 @@ -// Copyright 2018-2021 VMware, Inc. +// Copyright 2018-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* @@ -9,9 +9,7 @@ #pragma once -#include "platform.h" #include "allocator.h" -#include "io.h" #include "async.h" typedef struct page_handle { diff --git a/src/clockcache.c b/src/clockcache.c index d0e9ee922..3ea5e4c21 100644 --- a/src/clockcache.c +++ b/src/clockcache.c @@ -1,4 +1,4 @@ -// Copyright 2018-2021 VMware, Inc. +// Copyright 2018-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* @@ -8,13 +8,14 @@ * This file contains the implementation for a concurrent clock cache. *----------------------------------------------------------------------------- */ -#include "platform.h" - -#include "allocator.h" #include "clockcache.h" -#include "io.h" -#include -#include "util.h" +#include "platform_threads.h" +#include "platform_sleep.h" +#include "platform_typed_alloc.h" +#include "platform_time.h" +#include "platform_buffer.h" +#include "allocator.h" +#include "platform_io.h" #include "poison.h" diff --git a/src/clockcache.h b/src/clockcache.h index 5c76500dd..7f5bbdbbb 100644 --- a/src/clockcache.h +++ b/src/clockcache.h @@ -1,4 +1,4 @@ -// Copyright 2018-2021 VMware, Inc. +// Copyright 2018-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* @@ -9,9 +9,11 @@ #pragma once +#include "platform_buffer.h" +#include "platform_threads.h" #include "allocator.h" #include "cache.h" -#include "io.h" +#include "platform_io.h" // #define ADDR_TRACING #define TRACE_ADDR (UINT64_MAX - 1) diff --git a/src/core.c b/src/core.c index 967d91e7f..27b421e7e 100644 --- a/src/core.c +++ b/src/core.c @@ -1,4 +1,4 @@ -// Copyright 2018-2021 VMware, Inc. +// Copyright 2018-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* @@ -8,6 +8,8 @@ */ #include "core.h" +#include "platform_sleep.h" +#include "platform_time.h" #include "poison.h" #define LATENCYHISTO_SIZE 15 @@ -627,7 +629,7 @@ core_memtable_flush_internal(core_handle *spl, uint64 generation) } static void -core_memtable_flush_internal_virtual(void *arg, void *scratch) +core_memtable_flush_internal_virtual(void *arg) { core_memtable_args *mt_args = arg; core_memtable_flush_internal(mt_args->spl, mt_args->generation); @@ -1201,18 +1203,18 @@ core_insert(core_handle *spl, key tuple_key, message data) switch (message_class(data)) { case MESSAGE_TYPE_INSERT: spl->stats[tid].insertions++; - platform_histo_insert(spl->stats[tid].insert_latency_histo, - platform_timestamp_elapsed(ts)); + histogram_insert(spl->stats[tid].insert_latency_histo, + platform_timestamp_elapsed(ts)); break; case MESSAGE_TYPE_UPDATE: spl->stats[tid].updates++; - platform_histo_insert(spl->stats[tid].update_latency_histo, - platform_timestamp_elapsed(ts)); + histogram_insert(spl->stats[tid].update_latency_histo, + platform_timestamp_elapsed(ts)); break; case MESSAGE_TYPE_DELETE: spl->stats[tid].deletions++; - platform_histo_insert(spl->stats[tid].delete_latency_histo, - platform_timestamp_elapsed(ts)); + histogram_insert(spl->stats[tid].delete_latency_histo, + platform_timestamp_elapsed(ts)); break; default: platform_assert(0); @@ -1462,20 +1464,20 @@ core_create(core_config *cfg, platform_assert(spl->stats); for (uint64 i = 0; i < MAX_THREADS; i++) { platform_status rc; - rc = platform_histo_create(spl->heap_id, - LATENCYHISTO_SIZE + 1, - latency_histo_buckets, - &spl->stats[i].insert_latency_histo); + rc = histogram_create(spl->heap_id, + LATENCYHISTO_SIZE + 1, + latency_histo_buckets, + &spl->stats[i].insert_latency_histo); platform_assert_status_ok(rc); - rc = platform_histo_create(spl->heap_id, - LATENCYHISTO_SIZE + 1, - latency_histo_buckets, - &spl->stats[i].update_latency_histo); + rc = histogram_create(spl->heap_id, + LATENCYHISTO_SIZE + 1, + latency_histo_buckets, + &spl->stats[i].update_latency_histo); platform_assert_status_ok(rc); - rc = platform_histo_create(spl->heap_id, - LATENCYHISTO_SIZE + 1, - latency_histo_buckets, - &spl->stats[i].delete_latency_histo); + rc = histogram_create(spl->heap_id, + LATENCYHISTO_SIZE + 1, + latency_histo_buckets, + &spl->stats[i].delete_latency_histo); platform_assert_status_ok(rc); } } @@ -1536,20 +1538,20 @@ core_mount(core_config *cfg, platform_assert(spl->stats); for (uint64 i = 0; i < MAX_THREADS; i++) { platform_status rc; - rc = platform_histo_create(spl->heap_id, - LATENCYHISTO_SIZE + 1, - latency_histo_buckets, - &spl->stats[i].insert_latency_histo); + rc = histogram_create(spl->heap_id, + LATENCYHISTO_SIZE + 1, + latency_histo_buckets, + &spl->stats[i].insert_latency_histo); platform_assert_status_ok(rc); - rc = platform_histo_create(spl->heap_id, - LATENCYHISTO_SIZE + 1, - latency_histo_buckets, - &spl->stats[i].update_latency_histo); + rc = histogram_create(spl->heap_id, + LATENCYHISTO_SIZE + 1, + latency_histo_buckets, + &spl->stats[i].update_latency_histo); platform_assert_status_ok(rc); - rc = platform_histo_create(spl->heap_id, - LATENCYHISTO_SIZE + 1, - latency_histo_buckets, - &spl->stats[i].delete_latency_histo); + rc = histogram_create(spl->heap_id, + LATENCYHISTO_SIZE + 1, + latency_histo_buckets, + &spl->stats[i].delete_latency_histo); platform_assert_status_ok(rc); } } @@ -1604,12 +1606,9 @@ core_destroy(core_handle *spl) if (spl->cfg.use_stats) { for (uint64 i = 0; i < MAX_THREADS; i++) { - platform_histo_destroy(spl->heap_id, - &spl->stats[i].insert_latency_histo); - platform_histo_destroy(spl->heap_id, - &spl->stats[i].update_latency_histo); - platform_histo_destroy(spl->heap_id, - &spl->stats[i].delete_latency_histo); + histogram_destroy(spl->heap_id, &spl->stats[i].insert_latency_histo); + histogram_destroy(spl->heap_id, &spl->stats[i].update_latency_histo); + histogram_destroy(spl->heap_id, &spl->stats[i].delete_latency_histo); } platform_free(spl->heap_id, spl->stats); } @@ -1629,12 +1628,9 @@ core_unmount(core_handle **spl_in) trunk_context_deinit(&spl->trunk_context); if (spl->cfg.use_stats) { for (uint64 i = 0; i < MAX_THREADS; i++) { - platform_histo_destroy(spl->heap_id, - &spl->stats[i].insert_latency_histo); - platform_histo_destroy(spl->heap_id, - &spl->stats[i].update_latency_histo); - platform_histo_destroy(spl->heap_id, - &spl->stats[i].delete_latency_histo); + histogram_destroy(spl->heap_id, &spl->stats[i].insert_latency_histo); + histogram_destroy(spl->heap_id, &spl->stats[i].update_latency_histo); + histogram_destroy(spl->heap_id, &spl->stats[i].delete_latency_histo); } platform_free(spl->heap_id, spl->stats); } @@ -1715,26 +1711,26 @@ core_print_insertion_stats(platform_log_handle *log_handle, core_handle *spl) return; } - platform_histo_handle insert_lat_accum, update_lat_accum, delete_lat_accum; - platform_histo_create(spl->heap_id, + histogram_handle insert_lat_accum, update_lat_accum, delete_lat_accum; + histogram_create(spl->heap_id, LATENCYHISTO_SIZE + 1, latency_histo_buckets, &insert_lat_accum); - platform_histo_create(spl->heap_id, + histogram_create(spl->heap_id, LATENCYHISTO_SIZE + 1, latency_histo_buckets, &update_lat_accum); - platform_histo_create(spl->heap_id, + histogram_create(spl->heap_id, LATENCYHISTO_SIZE + 1, latency_histo_buckets, &delete_lat_accum); for (thr_i = 0; thr_i < MAX_THREADS; thr_i++) { - platform_histo_merge_in(insert_lat_accum, + histogram_merge_in(insert_lat_accum, spl->stats[thr_i].insert_latency_histo); - platform_histo_merge_in(update_lat_accum, + histogram_merge_in(update_lat_accum, spl->stats[thr_i].update_latency_histo); - platform_histo_merge_in(delete_lat_accum, + histogram_merge_in(delete_lat_accum, spl->stats[thr_i].delete_latency_histo); global->root_compactions += spl->stats[thr_i].root_compactions; @@ -1780,12 +1776,12 @@ core_print_insertion_stats(platform_log_handle *log_handle, core_handle *spl) platform_log(log_handle, "\n"); platform_log(log_handle, "Latency Histogram Statistics\n"); - platform_histo_print(insert_lat_accum, "Insert Latency Histogram (ns):", log_handle); - platform_histo_print(update_lat_accum, "Update Latency Histogram (ns):", log_handle); - platform_histo_print(delete_lat_accum, "Delete Latency Histogram (ns):", log_handle); - platform_histo_destroy(spl->heap_id, &insert_lat_accum); - platform_histo_destroy(spl->heap_id, &update_lat_accum); - platform_histo_destroy(spl->heap_id, &delete_lat_accum); + histogram_print(insert_lat_accum, "Insert Latency Histogram (ns):", log_handle); + histogram_print(update_lat_accum, "Update Latency Histogram (ns):", log_handle); + histogram_print(delete_lat_accum, "Delete Latency Histogram (ns):", log_handle); + histogram_destroy(spl->heap_id, &insert_lat_accum); + histogram_destroy(spl->heap_id, &update_lat_accum); + histogram_destroy(spl->heap_id, &delete_lat_accum); platform_log(log_handle, "Flush Statistics\n"); @@ -1921,30 +1917,30 @@ core_reset_stats(core_handle *spl) { if (spl->cfg.use_stats) { for (threadid thr_i = 0; thr_i < MAX_THREADS; thr_i++) { - platform_histo_destroy(spl->heap_id, - &spl->stats[thr_i].insert_latency_histo); - platform_histo_destroy(spl->heap_id, - &spl->stats[thr_i].update_latency_histo); - platform_histo_destroy(spl->heap_id, - &spl->stats[thr_i].delete_latency_histo); + histogram_destroy(spl->heap_id, + &spl->stats[thr_i].insert_latency_histo); + histogram_destroy(spl->heap_id, + &spl->stats[thr_i].update_latency_histo); + histogram_destroy(spl->heap_id, + &spl->stats[thr_i].delete_latency_histo); memset(&spl->stats[thr_i], 0, sizeof(spl->stats[thr_i])); platform_status rc; - rc = platform_histo_create(spl->heap_id, - LATENCYHISTO_SIZE + 1, - latency_histo_buckets, - &spl->stats[thr_i].insert_latency_histo); + rc = histogram_create(spl->heap_id, + LATENCYHISTO_SIZE + 1, + latency_histo_buckets, + &spl->stats[thr_i].insert_latency_histo); platform_assert_status_ok(rc); - rc = platform_histo_create(spl->heap_id, - LATENCYHISTO_SIZE + 1, - latency_histo_buckets, - &spl->stats[thr_i].update_latency_histo); + rc = histogram_create(spl->heap_id, + LATENCYHISTO_SIZE + 1, + latency_histo_buckets, + &spl->stats[thr_i].update_latency_histo); platform_assert_status_ok(rc); - rc = platform_histo_create(spl->heap_id, - LATENCYHISTO_SIZE + 1, - latency_histo_buckets, - &spl->stats[thr_i].delete_latency_histo); + rc = histogram_create(spl->heap_id, + LATENCYHISTO_SIZE + 1, + latency_histo_buckets, + &spl->stats[thr_i].delete_latency_histo); platform_assert_status_ok(rc); } } @@ -2002,9 +1998,3 @@ core_config_init(core_config *core_cfg, // When everything succeeds, return success. return STATUS_OK; } - -size_t -core_get_scratch_size() -{ - return 0; -} diff --git a/src/core.h b/src/core.h index 60a5e82b5..f756b4339 100644 --- a/src/core.h +++ b/src/core.h @@ -1,4 +1,4 @@ -// Copyright 2018-2021 VMware, Inc. +// Copyright 2018-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* @@ -13,6 +13,7 @@ #include "memtable.h" #include "log.h" #include "trunk.h" +#include "histogram.h" /* * Upper-bound on most number of branches that we can find our lookup-key in. @@ -52,9 +53,9 @@ typedef struct core_stats { uint64 updates; uint64 deletions; - platform_histo_handle insert_latency_histo; - platform_histo_handle update_latency_histo; - platform_histo_handle delete_latency_histo; + histogram_handle insert_latency_histo; + histogram_handle update_latency_histo; + histogram_handle delete_latency_histo; uint64 memtable_flushes; uint64 memtable_flush_time_ns; @@ -273,5 +274,3 @@ core_config_init(core_config *trunk_cfg, bool32 use_stats, bool32 verbose_logging, platform_log_handle *log_handle); -size_t -core_get_scratch_size(); diff --git a/src/data_internal.c b/src/data_internal.c index 5889defcc..d64c1fc5e 100644 --- a/src/data_internal.c +++ b/src/data_internal.c @@ -1,4 +1,8 @@ +// Copyright 2018-2026 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + #include "data_internal.h" +#include "poison.h" message_type merge_accumulator_message_class(const merge_accumulator *ma) diff --git a/src/data_internal.h b/src/data_internal.h index da77a6906..740e744f2 100644 --- a/src/data_internal.h +++ b/src/data_internal.h @@ -1,4 +1,4 @@ -// Copyright 2018-2021 VMware, Inc. +// Copyright 2018-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* diff --git a/src/default_data_config.c b/src/default_data_config.c index 7341d6356..87da136cf 100644 --- a/src/default_data_config.c +++ b/src/default_data_config.c @@ -1,4 +1,4 @@ -// Copyright 2018-2021 VMware, Inc. +// Copyright 2018-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 // A default data_config suitable for simple key/value applications @@ -6,11 +6,9 @@ // // This data_config does not support blind mutation operations -#include "platform.h" - #include "splinterdb/default_data_config.h" -#include "splinterdb/splinterdb.h" #include "util.h" +#include "platform_hash.h" #include "poison.h" diff --git a/src/histogram.c b/src/histogram.c new file mode 100644 index 000000000..112e4310c --- /dev/null +++ b/src/histogram.c @@ -0,0 +1,80 @@ +// Copyright 2018-2026 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +/* + * histogram.c -- + * + * This file contains the implementation for histogram data collection. + */ + +#include "histogram.h" +#include "platform_typed_alloc.h" +#include "platform_log.h" +#include "poison.h" + +platform_status +histogram_create(platform_heap_id heap_id, + uint32 num_buckets, + const int64 *const bucket_limits, + histogram_handle *histo) +{ + histogram_handle hh; + hh = TYPED_MANUAL_MALLOC(heap_id, + hh, + sizeof(hh) // NOLINT(bugprone-sizeof-expression) + + num_buckets * sizeof(hh->count[0])); + if (!hh) { + return STATUS_NO_MEMORY; + } + hh->num_buckets = num_buckets; + hh->bucket_limits = bucket_limits; + hh->total = 0; + hh->min = INT64_MAX; + hh->max = INT64_MIN; + hh->num = 0; + memset(hh->count, 0, hh->num_buckets * sizeof(hh->count[0])); + + *histo = hh; + return STATUS_OK; +} + +void +histogram_destroy(platform_heap_id heap_id, histogram_handle *histo_out) +{ + platform_assert(histo_out); + histogram_handle histo = *histo_out; + platform_free(heap_id, histo); + *histo_out = NULL; +} + +void +histogram_print(histogram_handle histo, + const char *name, + platform_log_handle *log_handle) +{ + if (histo->num == 0) { + return; + } + + platform_log(log_handle, "%s\n", name); + platform_log(log_handle, "min: %ld\n", histo->min); + platform_log(log_handle, "max: %ld\n", histo->max); + platform_log(log_handle, + "mean: %ld\n", + histo->num == 0 ? 0 : histo->total / histo->num); + platform_log(log_handle, "count: %ld\n", histo->num); + for (uint32 i = 0; i < histo->num_buckets; i++) { + if (i == histo->num_buckets - 1) { + platform_log(log_handle, + "%-12ld > %12ld\n", + histo->count[i], + histo->bucket_limits[i - 1]); + } else { + platform_log(log_handle, + "%-12ld <= %12ld\n", + histo->count[i], + histo->bucket_limits[i]); + } + } + platform_log(log_handle, "\n"); +} diff --git a/src/histogram.h b/src/histogram.h new file mode 100644 index 000000000..b1801205b --- /dev/null +++ b/src/histogram.h @@ -0,0 +1,91 @@ +// Copyright 2018-2026 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +/* + * histogram.h -- + * + * This file contains the interface for histogram data collection. + */ + + +#pragma once + +#include "splinterdb/platform_linux/public_platform.h" +#include "platform_heap.h" +#include "platform_assert.h" +#include "platform_status.h" + +typedef struct histogram { + unsigned int num_buckets; + const long *bucket_limits; + long min, max, total; + unsigned long num; // no. of elements + unsigned long count[]; +} *histogram_handle; + +platform_status +histogram_create(platform_heap_id heap_id, + uint32 num_buckets, + const int64 *const bucket_limits, + histogram_handle *histo); + +void +histogram_destroy(platform_heap_id heap_id, histogram_handle *histo); + +void +histogram_print(histogram_handle histo, + const char *name, + platform_log_handle *log_handle); + +static inline void +histogram_insert(histogram_handle histo, int64 datum) +{ + int lo = 0, hi = histo->num_buckets - 1; + + while (hi > lo) { + int mid = lo + (hi - lo) / 2; + + if (datum > histo->bucket_limits[mid]) { + lo = mid + 1; + } else { + hi = mid - 1; + } + } + platform_assert(lo < histo->num_buckets); + histo->count[lo]++; + if (histo->num == 0) { + histo->min = histo->max = datum; + } else { + histo->max = MAX(histo->max, datum); + histo->min = MIN(histo->min, datum); + } + histo->total += datum; + histo->num++; +} + +static inline void +histogram_merge_in(histogram_handle dest_histo, histogram_handle src_histo) +{ + uint32 i; + if (src_histo->num == 0) { + return; + } + + platform_assert(dest_histo->num_buckets == src_histo->num_buckets); + for (i = 0; i < dest_histo->num_buckets - 1; i++) { + platform_assert(dest_histo->bucket_limits[i] + == src_histo->bucket_limits[i]); + } + if (src_histo->min < dest_histo->min || dest_histo->num == 0) { + dest_histo->min = src_histo->min; + } + if (src_histo->max > dest_histo->max || dest_histo->num == 0) { + dest_histo->max = src_histo->max; + } + dest_histo->total += src_histo->total; + dest_histo->num += src_histo->num; + + for (i = 0; i < dest_histo->num_buckets; i++) { + dest_histo->count[i] += src_histo->count[i]; + } +} diff --git a/src/iterator.h b/src/iterator.h index 7e253ba2d..5c5546050 100644 --- a/src/iterator.h +++ b/src/iterator.h @@ -1,10 +1,9 @@ -// Copyright 2018-2021 VMware, Inc. +// Copyright 2018-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 #pragma once #include "data_internal.h" -#include "util.h" #include "vector.h" typedef struct iterator iterator; diff --git a/src/log.h b/src/log.h index 21ded8df8..325d07648 100644 --- a/src/log.h +++ b/src/log.h @@ -1,4 +1,4 @@ -// Copyright 2018-2021 VMware, Inc. +// Copyright 2018-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* @@ -9,9 +9,7 @@ #pragma once -#include "platform.h" #include "cache.h" -#include "util.h" #include "data_internal.h" typedef struct log_handle log_handle; diff --git a/src/memtable.c b/src/memtable.c index d17a2552c..59211991a 100644 --- a/src/memtable.c +++ b/src/memtable.c @@ -1,4 +1,4 @@ -// Copyright 2018-2021 VMware, Inc. +// Copyright 2018-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* @@ -9,8 +9,8 @@ *----------------------------------------------------------------------------- */ -#include "platform.h" #include "memtable.h" +#include "platform_sleep.h" #include "poison.h" @@ -46,71 +46,70 @@ memtable_process(memtable_context *ctxt, uint64 generation) static inline void memtable_begin_insert(memtable_context *ctxt) { - platform_batch_rwlock_get(ctxt->rwlock, MEMTABLE_INSERT_LOCK_IDX); + batch_rwlock_get(ctxt->rwlock, MEMTABLE_INSERT_LOCK_IDX); } void memtable_end_insert(memtable_context *ctxt) { - platform_batch_rwlock_unget(ctxt->rwlock, MEMTABLE_INSERT_LOCK_IDX); + batch_rwlock_unget(ctxt->rwlock, MEMTABLE_INSERT_LOCK_IDX); } static inline bool32 memtable_try_begin_insert_rotation(memtable_context *ctxt) { - if (!platform_batch_rwlock_try_claim(ctxt->rwlock, MEMTABLE_INSERT_LOCK_IDX)) - { + if (!batch_rwlock_try_claim(ctxt->rwlock, MEMTABLE_INSERT_LOCK_IDX)) { return FALSE; } - platform_batch_rwlock_lock(ctxt->rwlock, MEMTABLE_INSERT_LOCK_IDX); + batch_rwlock_lock(ctxt->rwlock, MEMTABLE_INSERT_LOCK_IDX); return TRUE; } static inline void memtable_end_insert_rotation(memtable_context *ctxt) { - platform_batch_rwlock_unlock(ctxt->rwlock, MEMTABLE_INSERT_LOCK_IDX); - platform_batch_rwlock_unclaim(ctxt->rwlock, MEMTABLE_INSERT_LOCK_IDX); + batch_rwlock_unlock(ctxt->rwlock, MEMTABLE_INSERT_LOCK_IDX); + batch_rwlock_unclaim(ctxt->rwlock, MEMTABLE_INSERT_LOCK_IDX); } static inline void memtable_begin_raw_rotation(memtable_context *ctxt) { - platform_batch_rwlock_get(ctxt->rwlock, MEMTABLE_INSERT_LOCK_IDX); - platform_batch_rwlock_claim_loop(ctxt->rwlock, MEMTABLE_INSERT_LOCK_IDX); - platform_batch_rwlock_lock(ctxt->rwlock, MEMTABLE_INSERT_LOCK_IDX); + batch_rwlock_get(ctxt->rwlock, MEMTABLE_INSERT_LOCK_IDX); + batch_rwlock_claim_loop(ctxt->rwlock, MEMTABLE_INSERT_LOCK_IDX); + batch_rwlock_lock(ctxt->rwlock, MEMTABLE_INSERT_LOCK_IDX); } static inline void memtable_end_raw_rotation(memtable_context *ctxt) { - platform_batch_rwlock_full_unlock(ctxt->rwlock, MEMTABLE_INSERT_LOCK_IDX); + batch_rwlock_full_unlock(ctxt->rwlock, MEMTABLE_INSERT_LOCK_IDX); } void memtable_begin_lookup(memtable_context *ctxt) { - platform_batch_rwlock_get(ctxt->rwlock, MEMTABLE_LOOKUP_LOCK_IDX); + batch_rwlock_get(ctxt->rwlock, MEMTABLE_LOOKUP_LOCK_IDX); } void memtable_end_lookup(memtable_context *ctxt) { - platform_batch_rwlock_unget(ctxt->rwlock, MEMTABLE_LOOKUP_LOCK_IDX); + batch_rwlock_unget(ctxt->rwlock, MEMTABLE_LOOKUP_LOCK_IDX); } void memtable_block_lookups(memtable_context *ctxt) { - platform_batch_rwlock_get(ctxt->rwlock, MEMTABLE_LOOKUP_LOCK_IDX); - platform_batch_rwlock_claim_loop(ctxt->rwlock, MEMTABLE_LOOKUP_LOCK_IDX); - platform_batch_rwlock_lock(ctxt->rwlock, MEMTABLE_LOOKUP_LOCK_IDX); + batch_rwlock_get(ctxt->rwlock, MEMTABLE_LOOKUP_LOCK_IDX); + batch_rwlock_claim_loop(ctxt->rwlock, MEMTABLE_LOOKUP_LOCK_IDX); + batch_rwlock_lock(ctxt->rwlock, MEMTABLE_LOOKUP_LOCK_IDX); } void memtable_unblock_lookups(memtable_context *ctxt) { - platform_batch_rwlock_full_unlock(ctxt->rwlock, MEMTABLE_LOOKUP_LOCK_IDX); + batch_rwlock_full_unlock(ctxt->rwlock, MEMTABLE_LOOKUP_LOCK_IDX); } @@ -315,7 +314,7 @@ memtable_context_create(platform_heap_id hid, platform_mutex_init( &ctxt->incorporation_mutex, platform_get_module_id(), hid); ctxt->rwlock = TYPED_MALLOC(hid, ctxt->rwlock); - platform_batch_rwlock_init(ctxt->rwlock); + batch_rwlock_init(ctxt->rwlock); for (uint64 mt_no = 0; mt_no < cfg->max_memtables; mt_no++) { uint64 generation = mt_no; diff --git a/src/memtable.h b/src/memtable.h index 4e1974036..39d3e6b1f 100644 --- a/src/memtable.h +++ b/src/memtable.h @@ -1,4 +1,4 @@ -// Copyright 2018-2021 VMware, Inc. +// Copyright 2018-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* @@ -9,10 +9,11 @@ #pragma once -#include "platform.h" +#include "platform_mutex.h" #include "task.h" #include "cache.h" #include "btree.h" +#include "batch_rwlock.h" #define MEMTABLE_SPACE_OVERHEAD_FACTOR (2) @@ -117,7 +118,7 @@ typedef struct memtable_context { // batch distributed read/write locks protect the generation and // generation_retired counters - platform_batch_rwlock *rwlock; + batch_rwlock *rwlock; // Protected by the MEMTABLE_INSERT_LOCK_IDX'th lock of rwlock. Can read // without lock. Must get read lock to freeze and write lock to modify. diff --git a/src/merge.c b/src/merge.c index 2131dcbcb..1ac02b2de 100644 --- a/src/merge.c +++ b/src/merge.c @@ -1,12 +1,12 @@ -// Copyright 2018-2021 VMware, Inc. +// Copyright 2018-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 -#include "platform.h" - #include "merge.h" #include "iterator.h" #include "util.h" - +#include "platform_sort.h" +#include "platform_log.h" +#include "platform_typed_alloc.h" #include "poison.h" /* This struct is just used to define type-safe flags. */ diff --git a/src/merge.h b/src/merge.h index 2cfea9553..bf8a76744 100644 --- a/src/merge.h +++ b/src/merge.h @@ -1,4 +1,4 @@ -// Copyright 2018-2021 VMware, Inc. +// Copyright 2018-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* @@ -11,7 +11,6 @@ #include "data_internal.h" #include "iterator.h" -#include "platform.h" // Hard limit tall tree range query? #define MAX_MERGE_ARITY (1024) diff --git a/src/mini_allocator.c b/src/mini_allocator.c index 850638546..9220519da 100644 --- a/src/mini_allocator.c +++ b/src/mini_allocator.c @@ -1,4 +1,4 @@ -// Copyright 2018-2021 VMware, Inc. +// Copyright 2018-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* @@ -10,12 +10,11 @@ *----------------------------------------------------------------------------- */ -#include "platform.h" - #include "allocator.h" #include "cache.h" #include "mini_allocator.h" #include "util.h" +#include "platform_sleep.h" #include "poison.h" @@ -39,7 +38,7 @@ typedef struct ONDISK mini_meta_hdr { char entry_buffer[]; } mini_meta_hdr; -#define TERMINAL_EXTENT_ADDR ((uint64)-1) +#define TERMINAL_EXTENT_ADDR ((uint64) - 1) /* *----------------------------------------------------------------------------- @@ -188,7 +187,6 @@ base_addr(cache *cc, uint64 addr) uint64 mini_init(mini_allocator *mini, cache *cc, - data_config *cfg, uint64 meta_head, uint64 meta_tail, uint64 num_batches, @@ -202,7 +200,6 @@ mini_init(mini_allocator *mini, ZERO_CONTENTS(mini); mini->cc = cc; mini->al = cache_get_allocator(cc); - mini->data_cfg = cfg; mini->meta_head = meta_head; mini->num_extents = 1; // for the meta page mini->num_batches = num_batches; diff --git a/src/mini_allocator.h b/src/mini_allocator.h index c5cd92cc7..9a2ac6bd7 100644 --- a/src/mini_allocator.h +++ b/src/mini_allocator.h @@ -1,4 +1,4 @@ -// Copyright 2018-2021 VMware, Inc. +// Copyright 2018-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* @@ -16,10 +16,8 @@ #pragma once -#include "platform.h" #include "allocator.h" #include "cache.h" -#include "data_internal.h" /* * Mini-allocator breaks extents into pages. The pages are fed out of separate @@ -36,7 +34,6 @@ typedef struct mini_allocator { allocator *al; cache *cc; - data_config *data_cfg; bool32 pinned; uint64 meta_head; volatile uint64 meta_tail; @@ -51,7 +48,6 @@ typedef struct mini_allocator { uint64 mini_init(mini_allocator *mini, cache *cc, - data_config *cfg, uint64 meta_head, uint64 meta_tail, uint64 num_batches, diff --git a/src/pcq.h b/src/pcq.h index a4e3ac9e2..aa89b7488 100644 --- a/src/pcq.h +++ b/src/pcq.h @@ -1,4 +1,4 @@ -// Copyright 2018-2021 VMware, Inc. +// Copyright 2018-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* @@ -10,7 +10,10 @@ #pragma once -#include "platform.h" +#include "splinterdb/public_platform.h" +#include "platform_machine.h" +#include "platform_heap.h" +#include "platform_typed_alloc.h" typedef struct { uint32 num_elems; diff --git a/src/async.h b/src/platform_linux/async.h similarity index 99% rename from src/async.h rename to src/platform_linux/async.h index 0c2af31b8..92ffee94e 100644 --- a/src/async.h +++ b/src/platform_linux/async.h @@ -1,4 +1,4 @@ -// Copyright 2024 VMware, Inc. +// Copyright 2024-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* @@ -127,6 +127,8 @@ #pragma once +#include "platform_assert.h" + /* Async functions return async_status. ASYNC_STATUS_RUNNING means that the * function has not yet completed. ASYNC_STATUS_DONE means that the function * has completed. Note that completion does not mean that the function diff --git a/src/platform_linux/laio.c b/src/platform_linux/laio.c index 3c030f93f..a7cb6cbe2 100644 --- a/src/platform_linux/laio.c +++ b/src/platform_linux/laio.c @@ -1,4 +1,4 @@ -// Copyright 2018-2021 VMware, Inc. +// Copyright 2018-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* @@ -17,11 +17,13 @@ * members using laio_get_metadata() and laio_get_iovec(). */ +#include "platform_status.h" #define POISON_FROM_PLATFORM_IMPLEMENTATION -#include "platform.h" -#include "async.h" #include "laio.h" +#include "platform_typed_alloc.h" +#include "async.h" +#include "platform_log.h" #include #include #include @@ -34,26 +36,12 @@ #endif #include +#define LAIO_CLEANER_TERMINATE_CALLBACK NULL + /* * Context management */ -static void -lock_ctx(laio_handle *io) -{ - while (__sync_lock_test_and_set(&io->ctx_lock, 1)) { - while (io->ctx_lock) { - platform_pause(); - } - } -} - -static void -unlock_ctx(laio_handle *io) -{ - __sync_lock_release(&io->ctx_lock); -} - static int laio_cleanup_one(io_process_context *pctx, int mincnt) { @@ -61,12 +49,12 @@ laio_cleanup_one(io_process_context *pctx, int mincnt) int status; status = io_getevents(pctx->ctx, mincnt, 1, &event, NULL); - if (status < 0 && !pctx->shutting_down) { + if (status < 0 && pctx->state != PROCESS_CONTEXT_STATE_SHUTTING_DOWN) { platform_error_log("%s(): OS-pid=%d, " "io_count=%lu," "failed with errorno=%d: %s\n", __func__, - platform_getpid(), + platform_get_os_pid(), pctx->io_count, -status, strerror(-status)); @@ -75,15 +63,23 @@ laio_cleanup_one(io_process_context *pctx, int mincnt) return 0; } - __sync_fetch_and_sub(&pctx->io_count, 1); + io_callback_t callback = (io_callback_t)event.data; + + // When we are being shut down, the main thread will set the callback to this + // value to indicate that we should exit. + if (callback == LAIO_CLEANER_TERMINATE_CALLBACK) { + __sync_fetch_and_sub(&pctx->io_count, 1); + return 0; + } // Invoke the callback for the one event that completed. - io_callback_t callback = (io_callback_t)event.data; callback(pctx->ctx, event.obj, event.res, 0); // Release one waiter if there is one async_wait_queue_release_one(&pctx->submit_waiters); + __sync_fetch_and_sub(&pctx->io_count, 1); + return 1; } @@ -92,59 +88,12 @@ laio_cleaner(void *arg) { io_process_context *pctx = (io_process_context *)arg; prctl(PR_SET_NAME, "laio_cleaner", 0, 0, 0); - while (!pctx->shutting_down) { + while (pctx->state != PROCESS_CONTEXT_STATE_SHUTTING_DOWN) { laio_cleanup_one(pctx, 1); } return NULL; } -/* - * Find the index of the IO context for this thread. If it doesn't exist, - * create it. - */ -static uint64 -get_ctx_idx(laio_handle *io) -{ - const pid_t pid = platform_getpid(); - - lock_ctx(io); - - for (int i = 0; i < MAX_THREADS; i++) { - if (io->ctx[i].pid == pid) { - io->ctx[i].thread_count++; - unlock_ctx(io); - return i; - } - } - - for (int i = 0; i < MAX_THREADS; i++) { - if (io->ctx[i].pid == 0) { - int status = io_setup(io->cfg->kernel_queue_size, &io->ctx[i].ctx); - if (status != 0) { - platform_error_log( - "io_setup() failed for PID=%d, ctx=%p with error=%d: %s\n", - pid, - &io->ctx[i].ctx, - -status, - strerror(-status)); - unlock_ctx(io); - return INVALID_TID; - } - io->ctx[i].pid = pid; - io->ctx[i].thread_count = 1; - io->ctx[i].shutting_down = 0; - async_wait_queue_init(&io->ctx[i].submit_waiters); - pthread_create( - &io->ctx[i].io_cleaner, NULL, laio_cleaner, &io->ctx[i]); - unlock_ctx(io); - return i; - } - } - - unlock_ctx(io); - return INVALID_TID; -} - /* * laio_read() - Basically a wrapper around pread(). */ @@ -188,14 +137,36 @@ laio_write(io_handle *ioh, void *buf, uint64 bytes, uint64 addr) * Accessor method: Return opaque handle to IO-context setup by io_setup(). */ static io_process_context * -laio_get_thread_context(io_handle *ioh) +laio_get_thread_context(laio_handle *io) { - laio_handle *io = (laio_handle *)ioh; - threadid tid = platform_get_tid(); - platform_assert(tid < MAX_THREADS, "Invalid tid=%lu", tid); - platform_assert( - io->ctx_idx[tid] < MAX_THREADS, "Invalid ctx_idx=%lu", io->ctx_idx[tid]); - return &io->ctx[io->ctx_idx[tid]]; + threadid pid = platform_linux_get_pid(); + platform_assert(pid < MAX_THREADS, "Invalid pid=%lu", pid); + if (io->ctx[pid].state == PROCESS_CONTEXT_STATE_UNINITIALIZED) { + while (__sync_lock_test_and_set(&io->ctx[pid].lock, 1)) { + // spin + } + if (io->ctx[pid].state != PROCESS_CONTEXT_STATE_UNINITIALIZED) { + __sync_lock_release(&io->ctx[pid].lock); + return &io->ctx[pid]; + } + int status = io_setup(io->cfg->kernel_queue_size, &io->ctx[pid].ctx); + if (status != 0) { + platform_error_log( + "io_setup() failed for PID=%lu, ctx=%p with error=%d: %s\n", + pid, + &io->ctx[pid].ctx, + -status, + strerror(-status)); + __sync_lock_release(&io->ctx[pid].lock); + return NULL; + } + io->ctx[pid].state = PROCESS_CONTEXT_STATE_INITIALIZED; + async_wait_queue_init(&io->ctx[pid].submit_waiters); + pthread_create( + &io->ctx[pid].io_cleaner, NULL, laio_cleaner, &io->ctx[pid]); + __sync_lock_release(&io->ctx[pid].lock); + } + return &io->ctx[pid]; } typedef struct laio_async_state { @@ -211,7 +182,6 @@ typedef struct laio_async_state { platform_status rc; struct iocb req; struct iocb *reqs[1]; - uint64 ctx_idx; int status; uint64 iovlen; struct iovec *iovs; @@ -287,7 +257,7 @@ laio_async_run(io_async_state *gios) async_return(ios); } - ios->pctx = laio_get_thread_context((io_handle *)ios->io); + ios->pctx = laio_get_thread_context(ios->io); if (ios->cmd == io_async_preadv) { io_prep_preadv(&ios->req, ios->io->fd, ios->iovs, ios->iovlen, ios->addr); } else { @@ -361,7 +331,7 @@ laio_async_run(io_async_state *gios) platform_error_log("%s(): OS-pid=%d, tid=%lu" ", io_submit errorno=%d: %s\n", __func__, - platform_getpid(), + platform_get_os_pid(), platform_get_tid(), -submit_status, strerror(-submit_status)); @@ -465,11 +435,9 @@ laio_cleanup(io_handle *ioh, uint64 count) { laio_handle *io = (laio_handle *)ioh; - threadid tid = platform_get_tid(); - platform_assert(tid < MAX_THREADS, "Invalid tid=%lu", tid); - platform_assert( - io->ctx_idx[tid] < MAX_THREADS, "Invalid ctx_idx=%lu", io->ctx_idx[tid]); - io_process_context *pctx = &io->ctx[io->ctx_idx[tid]]; + threadid pid = platform_linux_get_pid(); + platform_assert(pid < MAX_THREADS, "Invalid pid=%lu", pid); + io_process_context *pctx = &io->ctx[pid]; // Check for completion of up to 'count' events, one event at a time. // Or, check for all outstanding events (count == 0) @@ -491,7 +459,7 @@ laio_wait_all(io_handle *ioh) io = (laio_handle *)ioh; for (i = 0; i < MAX_THREADS; i++) { - if (io->ctx[i].pid == getpid()) { + if (i == platform_linux_get_pid()) { io_cleanup(ioh, 0); } else { while (0 < io->ctx[i].io_count) { @@ -501,80 +469,102 @@ laio_wait_all(io_handle *ioh) } } -/* - * When a thread registers with Splinter's task system, setup its - * IO-setup opaque handle that will be used by Async IO interfaces. - */ -static void -laio_register_thread(io_handle *ioh) +static platform_status +laio_wakeup_cleaner(io_process_context *pctx, + struct iocb *iocb, + int fd, + char buf[1]) { - const threadid tid = platform_get_tid(); - laio_handle *io = (laio_handle *)ioh; - uint64 idx = get_ctx_idx(io); - platform_assert( - (idx != INVALID_TID), "Failed to register IO for thread ID=%lu\n", tid); - io->ctx_idx[tid] = idx; + __sync_fetch_and_add(&pctx->io_count, 1); + io_prep_pread(iocb, fd, buf, 1, 0); + io_set_callback(iocb, LAIO_CLEANER_TERMINATE_CALLBACK); + int status = io_submit(pctx->ctx, 1, &iocb); + if (status != 1) { + return STATUS_IO_ERROR; + } + return STATUS_OK; } + static void -laio_deregister_thread(io_handle *ioh) +laio_process_termination_callback(threadid pid, void *arg) { - laio_handle *io = (laio_handle *)ioh; - io_process_context *pctx = laio_get_thread_context(ioh); - - platform_assert((pctx != NULL), - "Attempting to deregister IO for thread ID=%lu" - " found an uninitialized IO-context handle.\n", - platform_get_tid()); + laio_handle *io = (laio_handle *)arg; + io_process_context *pctx = &io->ctx[pid]; - // Process pending AIO-requests for this thread before deregistering it - laio_cleanup(ioh, 0); - - lock_ctx(io); - pctx->thread_count--; - if (pctx->thread_count == 0) { - pctx->shutting_down = TRUE; + if (pctx->state == PROCESS_CONTEXT_STATE_INITIALIZED) { + while (__sync_lock_test_and_set(&pctx->lock, 1)) { + // spin + } + if (pctx->state != PROCESS_CONTEXT_STATE_INITIALIZED) { + __sync_lock_release(&pctx->lock); + return; + } debug_assert(pctx->io_count == 0, "io_count=%lu", pctx->io_count); + + pctx->state = PROCESS_CONTEXT_STATE_SHUTTING_DOWN; + struct iocb iocb; + char buf[1]; + platform_status rc = laio_wakeup_cleaner(pctx, &iocb, io->fd, buf); + if (!SUCCESS(rc)) { + platform_error_log("laio_wakeup_cleaner() failed: %s\n", + strerror(errno)); + } + pthread_join(pctx->io_cleaner, NULL); int status = io_destroy(pctx->ctx); platform_assert(status == 0, "io_destroy() failed with error=%d: %s\n", -status, strerror(-status)); - pthread_join(pctx->io_cleaner, NULL); + async_wait_queue_deinit(&pctx->submit_waiters); + // subsequent io_setup calls on this ctx will fail if we don't reset it. // Seems like a bug in libaio/linux. - async_wait_queue_deinit(&pctx->submit_waiters); - memset(&pctx->ctx, 0, sizeof(pctx->ctx)); - pctx->pid = 0; + // + // Furthermore, clang-tidy generates a warning about the sizeof expression + // because pctx->ctx is a pointer and it is often a bug to memcpy onto a + // pointer (as opposed to memcpying to the location that the pointer + // points to). But here we want to use memcpy, rather than assigning + // NULL, because the type of pctx->ctx is supposed to be opaque. So we + // disable the warning. + memset(&pctx->ctx, + 0, + sizeof(pctx->ctx)); // NOLINT(bugprone-sizeof-expression) + + pctx->state = PROCESS_CONTEXT_STATE_UNINITIALIZED; + __sync_lock_release(&pctx->lock); } - unlock_ctx(io); } /* * Define an implementation of the abstract IO Ops interface methods. */ static io_ops laio_ops = { - .read = laio_read, - .write = laio_write, - .async_state_init = laio_async_state_init, - .cleanup = laio_cleanup, - .wait_all = laio_wait_all, - .register_thread = laio_register_thread, - .deregister_thread = laio_deregister_thread, + .read = laio_read, + .write = laio_write, + .async_state_init = laio_async_state_init, + .cleanup = laio_cleanup, + .wait_all = laio_wait_all, }; /* * Given an IO configuration, validate it. Allocate memory for various * sub-structures and allocate the SplinterDB device. Initialize the IO * sub-system, registering the file descriptor for SplinterDB device. + * Returns a pointer to the created io_handle, or NULL on error. */ -platform_status -io_handle_init(laio_handle *io, io_config *cfg, platform_heap_id hid) +io_handle * +laio_handle_create(io_config *cfg, platform_heap_id hid) { // Validate IO-configuration parameters platform_status rc = laio_config_valid(cfg); if (!SUCCESS(rc)) { - return rc; + return NULL; + } + + laio_handle *io = TYPED_MALLOC(hid, io); + if (io == NULL) { + return NULL; } memset(io, 0, sizeof(*io)); @@ -591,41 +581,55 @@ io_handle_init(laio_handle *io, io_config *cfg, platform_heap_id hid) if (io->fd == -1) { platform_error_log( "open() '%s' failed: %s\n", cfg->filename, strerror(errno)); - return CONST_STATUS(errno); + platform_free(hid, io); + return NULL; } struct stat statbuf; int r = fstat(io->fd, &statbuf); if (r) { platform_error_log("fstat failed: %s\n", strerror(errno)); - return STATUS_IO_ERROR; + close(io->fd); + platform_free(hid, io); + return NULL; } if (S_ISREG(statbuf.st_mode) && statbuf.st_size < 128 * 1024) { r = fallocate(io->fd, 0, 0, 128 * 1024); if (r) { platform_error_log("fallocate failed: %s\n", strerror(errno)); - return STATUS_IO_ERROR; + close(io->fd); + platform_free(hid, io); + return NULL; } } + io->pecnode.termination = laio_process_termination_callback; + io->pecnode.arg = io; + platform_linux_add_process_event_callback(&io->pecnode); + // leave req_hand set to 0 - return STATUS_OK; + return (io_handle *)io; } /* * Dismantle the handle for the IO sub-system, close file and release memory. */ void -io_handle_deinit(laio_handle *io) +laio_handle_destroy(io_handle *ioh) { - int status; + laio_handle *io = (laio_handle *)ioh; + int status; + + platform_linux_remove_process_event_callback(&io->pecnode); + threadid pid = platform_linux_get_pid(); + laio_process_termination_callback(pid, ioh); for (int i = 0; i < MAX_THREADS; i++) { - if (io->ctx[i].pid != 0) { - platform_error_log("ERROR: io_handle_deinit(): IO context for PID=%d" + if (io->ctx[i].state != PROCESS_CONTEXT_STATE_UNINITIALIZED) { + platform_error_log("ERROR: io_handle_destroy(): IO context for PID=%d" " is still active.\n", - io->ctx[i].pid); + i); } } @@ -637,6 +641,8 @@ io_handle_deinit(laio_handle *io) strerror(errno)); } platform_assert(status == 0); + + platform_free(io->heap_id, io); } /* @@ -646,13 +652,13 @@ io_handle_deinit(laio_handle *io) static inline bool32 laio_config_valid_page_size(io_config *cfg) { - return (cfg->page_size == LAIO_DEFAULT_PAGE_SIZE); + return (cfg->page_size == IO_DEFAULT_PAGE_SIZE); } static inline bool32 laio_config_valid_extent_size(io_config *cfg) { - return (cfg->extent_size == LAIO_DEFAULT_EXTENT_SIZE); + return (cfg->extent_size == IO_DEFAULT_EXTENT_SIZE); } diff --git a/src/platform_linux/laio.h b/src/platform_linux/laio.h index 2f300ae74..2bbb7d4f1 100644 --- a/src/platform_linux/laio.h +++ b/src/platform_linux/laio.h @@ -1,4 +1,4 @@ -// Copyright 2018-2021 VMware, Inc. +// Copyright 2018-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* @@ -9,29 +9,25 @@ #pragma once -#include "io.h" +#include "platform_io.h" +#include "platform_threads.h" +#include "platform_status.h" +#include "async.h" #include -/* - * SplinterDB can be configured with different page-sizes, given by these - * min & max values. - */ -#define LAIO_MIN_PAGE_SIZE (4096) -#define LAIO_MAX_PAGE_SIZE (8192) - -#define LAIO_DEFAULT_PAGE_SIZE LAIO_MIN_PAGE_SIZE -#define LAIO_DEFAULT_PAGES_PER_EXTENT 32 -#define LAIO_DEFAULT_EXTENT_SIZE \ - (LAIO_DEFAULT_PAGES_PER_EXTENT * LAIO_DEFAULT_PAGE_SIZE) +typedef enum process_context_state { + PROCESS_CONTEXT_STATE_UNINITIALIZED, + PROCESS_CONTEXT_STATE_INITIALIZED, + PROCESS_CONTEXT_STATE_SHUTTING_DOWN, +} process_context_state; typedef struct io_process_context { - pid_t pid; - uint64 thread_count; - bool32 shutting_down; - uint64 io_count; // inflight ios - io_context_t ctx; - pthread_t io_cleaner; - async_wait_queue submit_waiters; + process_context_state state; + uint64 lock; + uint64 io_count; // inflight ios + io_context_t ctx; + pthread_t io_cleaner; + async_wait_queue submit_waiters; } io_process_context; /* @@ -40,12 +36,18 @@ typedef struct io_process_context { typedef struct laio_handle { io_handle super; io_config *cfg; - int ctx_lock; io_process_context ctx[MAX_THREADS]; - uint64 ctx_idx[MAX_THREADS]; platform_heap_id heap_id; int fd; // File descriptor to Splinter device/file. + process_event_callback_list_node pecnode; } laio_handle; platform_status laio_config_valid(io_config *cfg); + +io_handle * +laio_handle_create(io_config *cfg, platform_heap_id hid); + +// The IO system must be quiesced before calling this function. +void +laio_handle_destroy(io_handle *ioh); \ No newline at end of file diff --git a/src/platform_linux/pl_splinter_trace.h b/src/platform_linux/pl_splinter_trace.h deleted file mode 100644 index 6eea16fbb..000000000 --- a/src/platform_linux/pl_splinter_trace.h +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright 2018-2021 VMware, Inc. -// SPDX-License-Identifier: Apache-2.0 - -/* pl_splinter_trace.h - * code that handles tracing. - */ - -#ifndef PL_SPLINTER_TRACE_H -#define PL_SPLINTER_TRACE_H - -// For now tracing for userspace is empty. -// To decide how tracing will be done for userspace. -#define TraceSplinter(...) -#define TraceUrgentSplinter(...) -#define TRACESplinter(...) -#define TRACE_DEFINE_TOKEN_BUCKETS(...) - -#endif /* PL_SPLINTER_TRACE_H */ diff --git a/src/platform_linux/platform.c b/src/platform_linux/platform.c deleted file mode 100644 index e3ef7dccd..000000000 --- a/src/platform_linux/platform.c +++ /dev/null @@ -1,676 +0,0 @@ -// Copyright 2018-2021 VMware, Inc. -// SPDX-License-Identifier: Apache-2.0 - -#include -#include -#include "platform.h" -#include "shmem.h" - -__thread threadid xxxtid = INVALID_TID; - -bool32 platform_use_hugetlb = FALSE; -bool32 platform_use_mlock = FALSE; - -// By default, platform_default_log() messages are sent to /dev/null -// and platform_error_log() messages go to stderr (see below). -// -// Use platform_set_log_streams() to send the log messages elsewhere. -platform_log_handle *Platform_default_log_handle = NULL; -platform_log_handle *Platform_error_log_handle = NULL; - -/* - * Declare globals to track heap handle/ID that may have been created when - * using shared memory. We stash away these handles so that we can return the - * right handle via platform_get_heap_id() interface, in case shared segments - * are in use. - */ -platform_heap_id Heap_id = NULL; - -// This function is run automatically at library-load time -void __attribute__((constructor)) platform_init_log_file_handles(void) -{ - FILE *dev_null_file = fopen("/dev/null", "w"); - platform_assert(dev_null_file != NULL); - - Platform_default_log_handle = dev_null_file; - Platform_error_log_handle = stderr; -} - -// Set the streams where informational and error messages will be printed. -void -platform_set_log_streams(platform_log_handle *info_stream, - platform_log_handle *error_stream) -{ - platform_assert(info_stream != NULL); - platform_assert(error_stream != NULL); - Platform_default_log_handle = info_stream; - Platform_error_log_handle = error_stream; -} - -// Return the stdout log-stream handle -platform_log_handle * -platform_get_stdout_stream(void) -{ - return Platform_default_log_handle; -} - -/* - * platform_heap_create() - Create a heap for memory allocation. - * - * By default, we just revert to process' heap-memory and use malloc() / free() - * for memory management. If Splinter is run with shared-memory configuration, - * create a shared-segment which acts as the 'heap' for memory allocation. - */ -platform_status -platform_heap_create(platform_module_id UNUSED_PARAM(module_id), - size_t max, - bool use_shmem, - platform_heap_id *heap_id) -{ - *heap_id = PROCESS_PRIVATE_HEAP_ID; - - if (use_shmem) { - platform_status rc = platform_shmcreate(max, heap_id); - if (SUCCESS(rc)) { - Heap_id = *heap_id; - } - return rc; - } - *heap_id = NULL; - return STATUS_OK; -} - -void -platform_heap_destroy(platform_heap_id *heap_id) -{ - // If shared segment was allocated, it's being tracked thru heap ID. - if (*heap_id) { - return platform_shmdestroy(heap_id); - } -} - -/* - * Certain modules, e.g. the buffer cache, need a very large buffer which - * may not be serviceable by the heap. Create the requested buffer using - * mmap() and initialize the input 'bh' to track this memory allocation. - */ -platform_status -platform_buffer_init(buffer_handle *bh, size_t length) -{ - platform_status rc = STATUS_NO_MEMORY; - - int prot = PROT_READ | PROT_WRITE; - - // Technically, for threaded execution model, MAP_PRIVATE is sufficient. - // And we only need to create this mmap()'ed buffer in MAP_SHARED for - // process-execution mode. But, at this stage, we don't know apriori if - // we will be using SplinterDB in a multi-process execution environment. - // So, always create this in SHARED mode. This still works for multiple - // threads. - int flags = MAP_SHARED | MAP_ANONYMOUS | MAP_NORESERVE; - if (platform_use_hugetlb) { - flags |= MAP_HUGETLB; - } - - bh->addr = mmap(NULL, length, prot, flags, -1, 0); - if (bh->addr == MAP_FAILED) { - platform_error_log( - "mmap (%lu bytes) failed with error: %s\n", length, strerror(errno)); - goto error; - } - - if (platform_use_mlock) { - int mlock_rv = mlock(bh->addr, length); - if (mlock_rv != 0) { - platform_error_log("mlock (%lu bytes) failed with error: %s\n", - length, - strerror(errno)); - // Save off rc from mlock()-failure, so we return that as status - rc = CONST_STATUS(errno); - - int rv = munmap(bh->addr, length); - if (rv != 0) { - // We are losing rc from this failure; can't return 2 statuses - platform_error_log("munmap %p (%lu bytes) failed with error: %s\n", - bh->addr, - length, - strerror(errno)); - } - goto error; - } - } - bh->length = length; - return STATUS_OK; - -error: - // Reset, in case mmap() or mlock() failed. - bh->addr = NULL; - return rc; -} - -void * -platform_buffer_getaddr(const buffer_handle *bh) -{ - return bh->addr; -} - -/* - * platform_buffer_deinit() - Deinit the buffer handle, which involves - * unmapping the memory for the large buffer created using mmap(). - * This is expected to be called on a 'bh' that has been successfully - * initialized, thru a prior platform_buffer_init() call. - */ -platform_status -platform_buffer_deinit(buffer_handle *bh) -{ - int ret; - ret = munmap(bh->addr, bh->length); - if (ret) { - return CONST_STATUS(errno); - } - - bh->addr = NULL; - bh->length = 0; - return STATUS_OK; -} - -/* - * platform_thread_create() - External interface to create a Splinter thread. - */ -platform_status -platform_thread_create(platform_thread *thread, - bool32 detached, - platform_thread_worker worker, - void *arg, - platform_heap_id UNUSED_PARAM(heap_id)) -{ - int ret; - - if (detached) { - pthread_attr_t attr; - pthread_attr_init(&attr); - size_t stacksize = 16UL * 1024UL; - pthread_attr_setstacksize(&attr, stacksize); - pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); - ret = pthread_create(thread, &attr, (void *(*)(void *))worker, arg); - pthread_attr_destroy(&attr); - } else { - ret = pthread_create(thread, NULL, (void *(*)(void *))worker, arg); - } - - return CONST_STATUS(ret); -} - -platform_status -platform_thread_join(platform_thread thread) -{ - int ret; - void *retval; - - ret = pthread_join(thread, &retval); - - return CONST_STATUS(ret); -} - -platform_thread -platform_thread_id_self() -{ - return pthread_self(); -} - -platform_status -platform_mutex_init(platform_mutex *lock, - platform_module_id UNUSED_PARAM(module_id), - platform_heap_id heap_id) -{ - platform_status status; - // Init mutex so it can be shared between processes, if so configured - pthread_mutexattr_t mattr; - pthread_mutexattr_init(&mattr); - status.r = pthread_mutexattr_setpshared(&mattr, PTHREAD_PROCESS_SHARED); - if (!SUCCESS(status)) { - return status; - } - - // clang-format off - status.r = pthread_mutex_init(&lock->mutex, - ((heap_id == PROCESS_PRIVATE_HEAP_ID) - ? NULL : &mattr)); - // clang-format on - - // Mess with output vars only in case of a success - if (SUCCESS(status)) { - lock->owner = INVALID_TID; - } - return status; -} - -platform_status -platform_mutex_destroy(platform_mutex *lock) -{ - // Cannot call destroy on a locked lock - platform_assert(lock->owner == INVALID_TID); - int ret = pthread_mutex_destroy(&lock->mutex); - return CONST_STATUS(ret); -} - -platform_status -platform_spinlock_init(platform_spinlock *lock, - platform_module_id UNUSED_PARAM(module_id), - platform_heap_id heap_id) -{ - int ret; - - // Init spinlock so it can be shared between processes, if so configured - ret = pthread_spin_init(lock, - ((heap_id == PROCESS_PRIVATE_HEAP_ID) - ? PTHREAD_PROCESS_PRIVATE - : PTHREAD_PROCESS_SHARED)); - return CONST_STATUS(ret); -} - -platform_status -platform_spinlock_destroy(platform_spinlock *lock) -{ - int ret; - - ret = pthread_spin_destroy(lock); - - return CONST_STATUS(ret); -} - -/* - *----------------------------------------------------------------------------- - * Batch Read/Write Locks - * - * These are generic distributed reader/writer locks with an intermediate claim - * state. These offer similar semantics to the locks in the clockcache, but in - * these locks read locks cannot be held with write locks. - * - * These locks are allocated in batches of PLATFORM_CACHELINE_SIZE / 2, so that - * the write lock and claim bits, as well as the distributed read counters, can - * be colocated across the batch. - *----------------------------------------------------------------------------- - */ - -void -platform_batch_rwlock_init(platform_batch_rwlock *lock) -{ - ZERO_CONTENTS(lock); -} - -void -platform_batch_rwlock_deinit(platform_batch_rwlock *lock) -{ - ZERO_CONTENTS(lock); -} - -/* - *----------------------------------------------------------------------------- - * lock/unlock - * - * Drains and blocks all gets (read locks) - * - * Caller must hold a claim - * Caller cannot hold a get (read lock) - *----------------------------------------------------------------------------- - */ - -void -platform_batch_rwlock_lock(platform_batch_rwlock *lock, uint64 lock_idx) -{ - platform_assert(lock->write_lock[lock_idx].claim); - debug_only uint8 was_locked = - __sync_lock_test_and_set(&lock->write_lock[lock_idx].lock, 1); - debug_assert(!was_locked, - "platform_batch_rwlock_lock: Attempt to lock a locked page.\n"); - - uint64 wait = 1; - for (uint64 i = 0; i < MAX_THREADS; i++) { - while (lock->read_counter[i][lock_idx] != 0) { - platform_sleep_ns(wait); - wait = wait > 2048 ? wait : 2 * wait; - } - wait = 1; - } -} - -void -platform_batch_rwlock_unlock(platform_batch_rwlock *lock, uint64 lock_idx) -{ - __sync_lock_release(&lock->write_lock[lock_idx].lock); -} - -void -platform_batch_rwlock_full_unlock(platform_batch_rwlock *lock, uint64 lock_idx) -{ - platform_batch_rwlock_unlock(lock, lock_idx); - platform_batch_rwlock_unclaim(lock, lock_idx); - platform_batch_rwlock_unget(lock, lock_idx); -} - -/* - *----------------------------------------------------------------------------- - * try_claim/claim/unlock - * - * A claim blocks all other claimants (and therefore all other writelocks, - * because writelocks are required to hold a claim during the writelock). - * - * Must hold a get (read lock) - * try_claim returns whether the claim succeeded - *----------------------------------------------------------------------------- - */ - -bool32 -platform_batch_rwlock_try_claim(platform_batch_rwlock *lock, uint64 lock_idx) -{ - threadid tid = platform_get_tid(); - debug_assert(lock->read_counter[tid][lock_idx]); - if (__sync_lock_test_and_set(&lock->write_lock[lock_idx].claim, 1)) { - return FALSE; - } - debug_only uint8 old_counter = - __sync_fetch_and_sub(&lock->read_counter[tid][lock_idx], 1); - debug_assert(0 < old_counter); - return TRUE; -} - -void -platform_batch_rwlock_claim_loop(platform_batch_rwlock *lock, uint64 lock_idx) -{ - uint64 wait = 1; - while (!platform_batch_rwlock_try_claim(lock, lock_idx)) { - platform_batch_rwlock_unget(lock, lock_idx); - platform_sleep_ns(wait); - wait = wait > 2048 ? wait : 2 * wait; - platform_batch_rwlock_get(lock, lock_idx); - } -} - -void -platform_batch_rwlock_unclaim(platform_batch_rwlock *lock, uint64 lock_idx) -{ - threadid tid = platform_get_tid(); - __sync_fetch_and_add(&lock->read_counter[tid][lock_idx], 1); - __sync_lock_release(&lock->write_lock[lock_idx].claim); -} - -/* - *----------------------------------------------------------------------------- - * get/unget - * - * Acquire a read lock - *----------------------------------------------------------------------------- - */ - -void -platform_batch_rwlock_get(platform_batch_rwlock *lock, uint64 lock_idx) -{ - threadid tid = platform_get_tid(); - while (1) { - uint64 wait = 1; - while (lock->write_lock[lock_idx].lock) { - platform_sleep_ns(wait); - wait = wait > 2048 ? wait : 2 * wait; - } - debug_only uint8 old_counter = - __sync_fetch_and_add(&lock->read_counter[tid][lock_idx], 1); - debug_assert(old_counter == 0); - if (!lock->write_lock[lock_idx].lock) { - return; - } - old_counter = __sync_fetch_and_sub(&lock->read_counter[tid][lock_idx], 1); - debug_assert(old_counter == 1); - } - platform_assert(0); -} - -void -platform_batch_rwlock_unget(platform_batch_rwlock *lock, uint64 lock_idx) -{ - threadid tid = platform_get_tid(); - debug_only uint8 old_counter = - __sync_fetch_and_sub(&lock->read_counter[tid][lock_idx], 1); - debug_assert(old_counter == 1); -} - - -platform_status -platform_condvar_init(platform_condvar *cv, platform_heap_id heap_id) -{ - platform_status status; - bool32 pth_condattr_setpshared_failed = FALSE; - - // Init mutex so it can be shared between processes, if so configured - pthread_mutexattr_t mattr; - pthread_mutexattr_init(&mattr); - status.r = pthread_mutexattr_setpshared(&mattr, PTHREAD_PROCESS_SHARED); - if (!SUCCESS(status)) { - goto mutexattr_setpshared_failed; - } - - // clang-format off - status.r = pthread_mutex_init(&cv->lock, - ((heap_id == PROCESS_PRIVATE_HEAP_ID) - ? NULL : &mattr)); - // clang-format on - debug_only int rv = pthread_mutexattr_destroy(&mattr); - debug_assert(rv == 0); - - if (!SUCCESS(status)) { - goto mutex_init_failed; - } - - // Init condition so it can be shared between processes, if so configured - pthread_condattr_t cattr; - pthread_condattr_init(&cattr); - status.r = pthread_condattr_setpshared(&cattr, PTHREAD_PROCESS_SHARED); - if (!SUCCESS(status)) { - pth_condattr_setpshared_failed = TRUE; - goto condattr_setpshared_failed; - } - - // clang-format off - status.r = pthread_cond_init(&cv->cond, - ((heap_id == PROCESS_PRIVATE_HEAP_ID) - ? NULL : &cattr)); - // clang-format on - - rv = pthread_condattr_destroy(&cattr); - debug_assert(rv == 0); - - // Upon a failure, before exiting, release mutex init'ed above ... - if (!SUCCESS(status)) { - goto cond_init_failed; - } - - return status; - - int ret = 0; -cond_init_failed: -condattr_setpshared_failed: - ret = pthread_mutex_destroy(&cv->lock); - // Yikes! Even this failed. We will lose the prev errno ... - if (ret) { - platform_error_log("%s() failed with error: %s\n", - (pth_condattr_setpshared_failed - ? "pthread_condattr_setpshared" - : "pthread_cond_init"), - platform_status_to_string(status)); - - // Return most recent failure rc - return CONST_STATUS(ret); - } -mutex_init_failed: -mutexattr_setpshared_failed: - return status; -} - -platform_status -platform_condvar_wait(platform_condvar *cv) -{ - int status; - - status = pthread_cond_wait(&cv->cond, &cv->lock); - return CONST_STATUS(status); -} - -platform_status -platform_condvar_signal(platform_condvar *cv) -{ - int status; - - status = pthread_cond_signal(&cv->cond); - return CONST_STATUS(status); -} - -platform_status -platform_condvar_broadcast(platform_condvar *cv) -{ - int status; - - status = pthread_cond_broadcast(&cv->cond); - return CONST_STATUS(status); -} - -platform_status -platform_histo_create(platform_heap_id heap_id, - uint32 num_buckets, - const int64 *const bucket_limits, - platform_histo_handle *histo) -{ - platform_histo_handle hh; - hh = TYPED_MANUAL_MALLOC( - heap_id, hh, sizeof(hh) + num_buckets * sizeof(hh->count[0])); - if (!hh) { - return STATUS_NO_MEMORY; - } - hh->num_buckets = num_buckets; - hh->bucket_limits = bucket_limits; - hh->total = 0; - hh->min = INT64_MAX; - hh->max = INT64_MIN; - hh->num = 0; - memset(hh->count, 0, hh->num_buckets * sizeof(hh->count[0])); - - *histo = hh; - return STATUS_OK; -} - -void -platform_histo_destroy(platform_heap_id heap_id, - platform_histo_handle *histo_out) -{ - platform_assert(histo_out); - platform_histo_handle histo = *histo_out; - platform_free(heap_id, histo); - *histo_out = NULL; -} - -void -platform_histo_print(platform_histo_handle histo, - const char *name, - platform_log_handle *log_handle) -{ - if (histo->num == 0) { - return; - } - - platform_log(log_handle, "%s\n", name); - platform_log(log_handle, "min: %ld\n", histo->min); - platform_log(log_handle, "max: %ld\n", histo->max); - platform_log(log_handle, - "mean: %ld\n", - histo->num == 0 ? 0 : histo->total / histo->num); - platform_log(log_handle, "count: %ld\n", histo->num); - for (uint32 i = 0; i < histo->num_buckets; i++) { - if (i == histo->num_buckets - 1) { - platform_log(log_handle, - "%-12ld > %12ld\n", - histo->count[i], - histo->bucket_limits[i - 1]); - } else { - platform_log(log_handle, - "%-12ld <= %12ld\n", - histo->count[i], - histo->bucket_limits[i]); - } - } - platform_log(log_handle, "\n"); -} - -char * -platform_strtok_r(char *str, const char *delim, platform_strtok_ctx *ctx) -{ - return strtok_r(str, delim, &ctx->token_str); -} - -void -platform_sort_slow(void *base, - size_t nmemb, - size_t size, - platform_sort_cmpfn cmpfn, - void *cmparg, - void *temp) -{ - return qsort_r(base, nmemb, size, cmpfn, cmparg); -} - -/* - * platform_assert_false() - - * - * Platform-specific assert implementation, with support to print an optional - * message and arguments involved in the assertion failure. The caller-macro - * ensures that this function will only be called when the 'exprval' is FALSE; - * i.e. the assertion is failing. - */ -__attribute__((noreturn)) void -platform_assert_false(const char *filename, - int linenumber, - const char *functionname, - const char *expr, - const char *message, - ...) -{ - va_list varargs; - va_start(varargs, message); - - // Run-time assertion messages go to stderr. - platform_assert_msg(Platform_error_log_handle, - filename, - linenumber, - functionname, - expr, - message, - varargs); - va_end(varargs); - platform_error_log("\n"); - - abort(); -} - -/* - * Lower-level function to generate the assertion message, alone. - */ -void -platform_assert_msg(platform_log_handle *log_handle, - const char *filename, - int linenumber, - const char *functionname, - const char *expr, - const char *message, - va_list varargs) -{ - static char assert_msg_fmt[] = "OS-pid=%d, OS-tid=%lu, Thread-ID=%lu, " - "Assertion failed at %s:%d:%s(): \"%s\". "; - platform_log(log_handle, - assert_msg_fmt, - platform_getpid(), - gettid(), - platform_get_tid(), // SplinterDB's thread-ID (index) - filename, - linenumber, - functionname, - expr); - vfprintf(log_handle, message, varargs); -} diff --git a/src/platform_linux/platform.h b/src/platform_linux/platform.h index 1eb62303e..812ccc441 100644 --- a/src/platform_linux/platform.h +++ b/src/platform_linux/platform.h @@ -1,828 +1,30 @@ -// Copyright 2018-2021 VMware, Inc. +// Copyright 2018-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 -#ifndef PLATFORM_H -#define PLATFORM_H - -#include "splinterdb/public_platform.h" - -/* - * Platform directory is chosen via -I include options to compiler. - * e.g. linux compile includes platform_linux directory - * Sections: - * 1- Shared types/typedefs (or #includes) that don't rely on anything - * platform-specific - * 2- (#include)Platform-specific types/typedefs - * 3- Shared types/typedefs that rely on platform types - * There should not be any generic #includes here, but potentially #includes - * of headers defined in splinterdb cannot be included until now. - * 4- Shared function prototypes. - * Shared inline function prototypes (including prototypes for the - * platform-specific inline functions). - * 5- (#include)Implementation of platform-specific inline functions - * 6- Implementations of shared inline functions that aren't platform specific - * (These may be candidates to move outside of platform.h entirely) - * 7- (Not actually in this header) - * Non-inline platform-specific function implementations belong in - * /.c - */ +#pragma once #define PLATFORM_LINUX -/* - * Section 1: - * Shared types/typedefs that don't rely on anything platform-specific - */ -typedef int32 bool32; - -#if !defined(SPLINTER_DEBUG) -# define SPLINTER_DEBUG 0 -#else -# if SPLINTER_DEBUG != 0 && SPLINTER_DEBUG != 1 -# error SPLINTER_DEBUG not 0 or 1 -# endif -#endif - -// Helper function so that ARRAYSIZE can be used inside of _Static_assert -#define ASSERT_EXPR(condition, return_value) \ - (sizeof(char[(condition) ? 1 : -1]) ? (return_value) : (return_value)) - -/* - * Utility macro to test if an indexable expression is an array. - * Gives an error at compile time if the expression is not indexable. - * Only pointers and arrays are indexable. - * Expression value is 0 for pointer and 1 for array - */ -#define IS_ARRAY(x) \ - __builtin_choose_expr( \ - __builtin_types_compatible_p(typeof((x)[0])[], typeof(x)), 1, 0) - -/* - * Errors at compile time if you use ARRAY_SIZE() on a pointer. - * ARRAY_SIZE can be used inside of _Static_assert. - */ -#define ARRAY_SIZE(x) ASSERT_EXPR(IS_ARRAY(x), (sizeof(x) / sizeof((x)[0]))) - -/* - * MAX_THREADS is used primarily for convenience, where allocations made on a - * per-thread basis create an array with MAX_THREADS items, e.g. the - * trunk_stats field in trunk_handle. The task subsystem also uses a 64-bit - * bit-array to track thread IDs in use. This could be changed relatively - * easily if needed. - */ -#define MAX_THREADS (64) -#define INVALID_TID (MAX_THREADS) - -#define HASH_SEED (42) - -/* - * C11 and higher already supports native _Static_assert which has good - * compiler output on errors - * Since we are lower than C11, we need something that works. - */ -#if !defined(__STDC_VERSION__) || __STDC_VERSION__ < 201112l -# define _Static_assert(expr, str) \ - do { \ - typedef char oc_assert_fail[((expr) ? 1 : -1)] \ - __attribute__((__unused__)); \ - } while (0) -#endif - -// Data unit constants -#define KiB (1024UL) -#define MiB (KiB * 1024) -#define GiB (MiB * 1024) -#define TiB (GiB * 1024) - -// Convert 'x' in unit-specifiers to bytes -#define KiB_TO_B(x) ((x)*KiB) -#define MiB_TO_B(x) ((x)*MiB) -#define GiB_TO_B(x) ((x)*GiB) -#define TiB_TO_B(x) ((x)*TiB) - -// Convert 'x' in bytes to 'int'-value with unit-specifiers -#define B_TO_KiB(x) ((x) / KiB) -#define B_TO_MiB(x) ((x) / MiB) -#define B_TO_GiB(x) ((x) / GiB) -#define B_TO_TiB(x) ((x) / TiB) - -// For x bytes, returns as int the fractional portion modulo a unit-specifier -#define B_TO_KiB_FRACT(x) ((100 * ((x) % KiB)) / KiB) -#define B_TO_MiB_FRACT(x) ((100 * ((x) % MiB)) / MiB) -#define B_TO_GiB_FRACT(x) ((100 * ((x) % GiB)) / GiB) -#define B_TO_TiB_FRACT(x) ((100 * ((x) % TiB)) / TiB) - -// Time unit constants -#define THOUSAND (1000UL) -#define MILLION (THOUSAND * THOUSAND) -#define BILLION (THOUSAND * MILLION) - -#define USEC_TO_SEC(x) ((x) / MILLION) -#define USEC_TO_NSEC(x) ((x)*THOUSAND) -#define NSEC_TO_SEC(x) ((x) / BILLION) -#define NSEC_TO_MSEC(x) ((x) / MILLION) -#define NSEC_TO_USEC(x) ((x) / THOUSAND) -#define SEC_TO_MSEC(x) ((x)*THOUSAND) -#define SEC_TO_USEC(x) ((x)*MILLION) -#define SEC_TO_NSEC(x) ((x)*BILLION) - -#define MAX_STRING_LENGTH 256 - -typedef void (*platform_thread_worker)(void *); - -/* - * The comparator follows the same conventions as that of qsort(3). Ie. if: - * a>b: return 1 - * a - -// Platform status -typedef struct { - internal_platform_status r; -} platform_status; - -#define CONST_STATUS(status) ((const platform_status){.r = status}) - -typedef struct { - uint64 v; -} PLATFORM_CACHELINE_ALIGNED cache_aligned_uint64; -_Static_assert(sizeof(cache_aligned_uint64) == PLATFORM_CACHELINE_SIZE, - "Attribute set wrong"); -typedef struct { - uint32 v; -} PLATFORM_CACHELINE_ALIGNED cache_aligned_uint32; -_Static_assert(sizeof(cache_aligned_uint32) == PLATFORM_CACHELINE_SIZE, - "Attribute set wrong"); - -typedef struct { - char *token_str; - char *last_token; - int last_token_len; -} platform_strtok_ctx; - -extern bool32 platform_use_hugetlb; -extern bool32 platform_use_mlock; - - -/* - * Section 3: - * Shared types/typedefs that rely on platform-specific types/typedefs - * There should not be any generic #includes here, but potentially #includes - * of headers defined in splinterdb cannot be included until now. - */ -extern platform_log_handle *Platform_default_log_handle; -extern platform_log_handle *Platform_error_log_handle; - - -/* - * Section 4: - * Shared function declarations. - * Shared inline function declarations (including declarations for the - * platform-specific inline functions). - * Implementations of shared inline functions that aren't platform specific - * (These may be candidates to move outside of platform.h entirely) - */ -#if SPLINTER_DEBUG -# define debug_assert(expr, ...) platform_assert(expr, __VA_ARGS__) -# define debug_only __attribute__((__unused__)) -# define debug_code(...) __VA_ARGS__ -#else -# define debug_assert(expr, ...) -# define debug_only __attribute__((__unused__)) -# define debug_code(...) -#endif // SPLINTER_DEBUG - -#define platform_assert_status_ok(_s) platform_assert(SUCCESS(_s)); - -// hash functions -typedef uint32 (*hash_fn)(const void *input, size_t length, unsigned int seed); - -extern platform_heap_id Heap_id; - -/* - * Provide a tag for callers that do not want to use shared-memory allocation, - * when configured but want to fallback to default scheme of allocating - * process-private memory. Typically, this would default to malloc()/free(). - * (Clients that repeatedly allocate and free a large chunk of memory in some - * code path would want to use this tag.) - */ -#define PROCESS_PRIVATE_HEAP_ID (platform_heap_id) NULL - -/* - * ----------------------------------------------------------------------------- - * TYPED_MANUAL_MALLOC(), TYPED_MANUAL_ZALLOC() - - * TYPED_ARRAY_MALLOC(), TYPED_ARRAY_ZALLOC() - - * - * Utility macros to avoid common memory allocation / initialization mistakes. - * NOTE: ZALLOC variants will also memset allocated memory chunk to 0. - * - * Call-flow is: - * TYPED_MALLOC() - * -> TYPED_ARRAY_MALLOC() - * -> TYPED_MANUAL_MALLOC() -> platform_aligned_malloc() - * - * TYPED_ZALLOC() - * -> TYPED_ARRAY_ZALLOC() - * -> TYPED_MANUAL_ZALLOC() -> platform_aligned_zalloc() - * - * ----------------------------------------------------------------------------- - * Common mistake to make is: - * TYPE *foo = platform_malloc(sizeof(WRONG_TYPE)); - * or - * TYPE *foo; - * ... - * foo = platform_malloc(sizeof(WRONG_TYPE)); - * WRONG_TYPE may have been the correct type and the code changed, or you may - * have the wrong number of '*'s, or you may have copied and pasted and forgot - * to change the type/size. - * - * A useful pattern is: - * TYPE *foo = platform_malloc(sizeof(*foo)) - * but you can still cause a mistake by not typing the variable name correctly - * twice. - * - * We _could_ make a macro MALLOC_AND_SET for the above pattern, e.g. - * define MALLOC_AND_SET(x) (x) = platform_malloc(sizeof(*x)) - * but that is hard to read. - * - * The boilerplate of extra typing (e.g. remember to type x twice) isn't a big - * problem, it's just that you can get the types wrong and the compiler won't - * notice. - * We can keep the easy-to-read pattern of `x = function(x)` with typesafety by - * including a cast inside the macro. - * - * These macros let you avoid the avoid the mistakes by typing malloc. - * - * struct foo *foo_pointer = TYPED_MALLOC(foo_pointer); - * struct foo *foo_pointer2; - * foo_pointer2 = TYPED_MALLOC(foo_pointer2); - * - * To replace dynamic/vla arrays, use TYPED_ARRAY_MALLOC, e.g. - * struct foo array[X()]; - * becomes - * struct foo *array = TYPED_ARRAY_MALLOC(array, X()); - * ZALLOC versions will also memset to 0. - * - * All mallocs here are cache aligned. - * Consider the following types: - * struct unaligned_foo { - * cache_aligned_type bar; - * }; - * If you (unaligned) malloc a 'unaligned_foo', the compile is still allowed to - * assume that bar is properly cache aligned. It may do unsafe optimizations. - * One known unsafe optimization is turning memset(...,0,...) into avx - * instructions that crash if something is not aligned. - * - * The simplest solution is to simply align ALL mallocs. - * We do not do a sufficient number of mallocs for this to have any major - * problems. The slight additional memory usage (for tiny mallocs) should not - * matter. The potential minor perf hit should also not matter due to us - * slowly coalescing all mallocs anyway into either initialization or amortized - * situations. - * - * Alternative solutions are to be careful with mallocs, and/or make ALL structs - * be aligned. - * - * Another common use case is if you have a struct with a flexible array member. - * In that case you should use TYPED_FLEXIBLE_STRUCT_(M|Z)ALLOC - * - * If you are doing memory size calculation manually (e.g. if you're avoiding - * multiple mallocs by doing one larger malloc and setting pointers manually, - * or the data type has a something[] or something[0] at the end) you should - * instead use the TYPED_*ALLOC_MANUAL macros that allow you to provide the - * exact size. These macros currently assume (and in debug mode assert) that - * you will never malloc LESS than the struct/type size. - * - * DO NOT USE these macros to assign to a void*. The debug asserts will cause - * a compile error when debug is on. Assigning to a void* should be done by - * calling aligned_alloc manually (or create a separate macro) - * - * Parameters: - * hid - Platform heap-ID to allocate memory from. - * v - Structure to allocate memory for. - * n - Number of bytes of memory to allocate. - * ----------------------------------------------------------------------------- - */ -#define TYPED_MANUAL_MALLOC(hid, v, n) \ - ({ \ - debug_assert((n) >= sizeof(*(v))); \ - (typeof(v))platform_aligned_malloc(hid, \ - PLATFORM_CACHELINE_SIZE, \ - (n), \ - STRINGIFY(v), \ - __func__, \ - __FILE__, \ - __LINE__); \ - }) -#define TYPED_MANUAL_ZALLOC(hid, v, n) \ - ({ \ - debug_assert((n) >= sizeof(*(v))); \ - (typeof(v))platform_aligned_zalloc(hid, \ - PLATFORM_CACHELINE_SIZE, \ - (n), \ - STRINGIFY(v), \ - __func__, \ - __FILE__, \ - __LINE__); \ - }) - -/* - * TYPED_ALIGNED_MALLOC(), TYPED_ALIGNED_ZALLOC() - * - * Allocate memory for a typed structure at caller-specified alignment. - * These are similar to TYPED_MANUAL_MALLOC() & TYPED_MANUAL_ZALLOC() but with - * the difference that the alignment is caller-specified. - * - * Parameters: - * hid - Platform heap-ID to allocate memory from. - * a - Alignment needed for allocated memory. - * v - Structure to allocate memory for. - * n - Number of bytes of memory to allocate. - */ -#define TYPED_ALIGNED_MALLOC(hid, a, v, n) \ - ({ \ - debug_assert((n) >= sizeof(*(v))); \ - (typeof(v))platform_aligned_malloc( \ - hid, (a), (n), STRINGIFY(v), __func__, __FILE__, __LINE__); \ - }) -#define TYPED_ALIGNED_ZALLOC(hid, a, v, n) \ - ({ \ - debug_assert((n) >= sizeof(*(v))); \ - (typeof(v))platform_aligned_zalloc( \ - hid, (a), (n), STRINGIFY(v), __func__, __FILE__, __LINE__); \ - }) - -/* - * FLEXIBLE_STRUCT_SIZE(): Compute the size of a structure 'v' with a nested - * flexible array member, array_field_name, with 'n' members. - * - * Flexible array members don't necessarily start after sizeof(v) - * They can start within the padding at the end, so the correct size - * needed to allocate a struct with a flexible array member is the - * larger of sizeof(struct v) or (offset of flexible array + - * n*sizeof(arraymember)) - * - * The only reasonable static assert we can do is check that the flexible array - * member is actually an array. We cannot check size==0 (compile error), and - * since it doesn't necessarily start at the end we also cannot check - * offset==sizeof. - * - * Parameters: - * v - Structure to allocate memory for. - * array_field_name - Name of flexible array field nested in 'v' - * n - Number of members in array_field_name[]. - */ -#define FLEXIBLE_STRUCT_SIZE(v, array_field_name, n) \ - ({ \ - _Static_assert(IS_ARRAY((v)->array_field_name), \ - "flexible array members must be arrays"); \ - max_size_t(sizeof(*(v)), \ - (n) * sizeof((v)->array_field_name[0]) \ - + offsetof(typeof(*(v)), array_field_name)); \ - }) - -/* - * ----------------------------------------------------------------------------- - * TYPED_FLEXIBLE_STRUCT_MALLOC(), TYPED_FLEXIBLE_STRUCT_ZALLOC() - - * Allocate memory for a structure with a nested flexible array member. - * - * Parameters: - * hid - Platform heap-ID to allocate memory from. - * v - Structure to allocate memory for. - * array_field_name - Name of flexible array field nested in 'v' - * n - Number of members in array_field_name[]. - * ----------------------------------------------------------------------------- - */ -#define TYPED_FLEXIBLE_STRUCT_MALLOC(hid, v, array_field_name, n) \ - TYPED_MANUAL_MALLOC( \ - hid, (v), FLEXIBLE_STRUCT_SIZE((v), array_field_name, (n))) - -#define TYPED_FLEXIBLE_STRUCT_ZALLOC(hid, v, array_field_name, n) \ - TYPED_MANUAL_ZALLOC( \ - hid, (v), FLEXIBLE_STRUCT_SIZE((v), array_field_name, (n))) - -/* - * TYPED_ARRAY_MALLOC(), TYPED_ARRAY_ZALLOC() - * Allocate memory for an array of 'n' elements of structure 'v'. - * - * Parameters: - * hid - Platform heap-ID to allocate memory from. - * v - Structure to allocate memory for. - * n - Number of members of type 'v' in array. - */ -#define TYPED_ARRAY_MALLOC(hid, v, n) \ - TYPED_MANUAL_MALLOC(hid, (v), (n) * sizeof(*(v))) -#define TYPED_ARRAY_ZALLOC(hid, v, n) \ - TYPED_MANUAL_ZALLOC(hid, (v), (n) * sizeof(*(v))) - -/* - * TYPED_ARRAY_MALLOC(), TYPED_ARRAY_ZALLOC() - * Allocate memory for one element of structure 'v'. - * - * Parameters: - * hid - Platform heap-ID to allocate memory from. - * v - Structure to allocate memory for. - */ -#define TYPED_MALLOC(hid, v) TYPED_ARRAY_MALLOC(hid, v, 1) -#define TYPED_ZALLOC(hid, v) TYPED_ARRAY_ZALLOC(hid, v, 1) - -/* - * ----------------------------------------------------------------------------- - * Utility macros to clear memory - * They have similar usage/prevent similar mistakes to the TYPED_MALLOC - * kind of macros. - * They are not appropriate when you have a malloced array or when you've - * done non-obvious sized mallocs. - * - * Array/Pointer/Struct have different implementations cause the size - * and pointer calculations are different. - * Passing any one type to a clearing function of another type is likely - * to be a bug, so if the calling isn't perfect it will give a compile-time - * error. - * ----------------------------------------------------------------------------- - */ -/* - * Zero an array. - * Cause compile-time error if used on pointer or non-indexible variable - */ -#define ZERO_ARRAY(v) \ - do { \ - _Static_assert(IS_ARRAY(v), "Use of ZERO_ARRAY on non-array object"); \ - memset((v), 0, sizeof(v)); \ - } while (0) - -/* - * Zero a manual array (e.g. we malloced an array). - * Cause compile-time error if used on an array or non-indexible variable - */ -#define ZERO_CONTENTS_N(v, n) \ - do { \ - _Static_assert(!IS_ARRAY(v), "Use of ZERO_CONTENTS on array"); \ - debug_assert((v) != NULL); \ - memset((v), 0, (n) * sizeof(*(v))); \ - } while (0) - -/* - * Zero a non-array pointer (clears what the pointer points to). - * Cause compile-time error if used on an array or non-indexible variable - * - * Should not be used to zero out structs. Use ZERO_STRUCT instead. - * It is difficult to add compile errors when you pass structs here, but - * a debug compile is likely to compile error on the debug_assert. - */ -#define ZERO_CONTENTS(v) ZERO_CONTENTS_N((v), 1) - -/* - * Zero a struct. - * We want to give a compile-time error if v is not a struct, so we cannot - * use something like memset(&v, 0, sizeof(v)); Even C11 is not rich enough - * to determine if a variable is a struct, so we use syntax errors to catch - * it instead. - * - * Unfortunately C11/gnu11 syntax is only rich enough to cause errors on - * pointers and primitives, but not arrays. - * - * Note: We could take advantage of the syntax and still use memset like this: - * __builtin_choose_expr( - * 0, - * (typeof(v)) {}, - * memset(&(v), 0, sizeof(v))) - * however while the above still does prevent pointers and primitives, it will - * incorrectly initialize arrays (read: corrupt memory) if they happen to get - * passed in. - * Struct assignment properly initializes both arrays and structs: - * v = (typeof(v)) {} - * Memset for structs: - * memset(&v, 0, sizeof(v)) - * Memset for arrays: - * memset(v, 0, sizeof(v)) - * - * memset performance is affected by compiler and headers - * struct assignment performance is just affected by compiler - * It's not obvious that one has a performance advantage and since memset is - * often a compiler intrinsic it's likely to have the same performance. - * - * Note; This version would only work during declaration: - * (v) = {} - * Note; This version could work during declaration and a regular statement, - * but we force it as a statement to match the usage of ZERO_ARRAY/ZERO_POINTER - * (v) = (typeof(v)) {} - * - * This macro intentionally CANNOT be used during declaration - * (see ZERO_STRUCT_AT_DECL). - */ -#define ZERO_STRUCT(v) \ - do { \ - (v) = (typeof(v)){}; \ - } while (0) - -/* - * Zero a struct at declaration time. - * Equivalent to doing: - * struct foo s; - * ZERO_STRUCT(s); - * Usage example: - * struct foo ZERO_STRUCT_AT_DECL(s); - * See documentation for ZERO_STRUCT. - * Note: - * You can actually use ZERO_STRUCT_AT_DECL as a regular statement, - * but it is slightly less safe because the first v cannot be wrapped - * in parenthesis. - */ -#define ZERO_STRUCT_AT_DECL(v) \ - v = (typeof(v)) {} - -void -platform_sort_slow(void *base, - size_t nmemb, - size_t size, - platform_sort_cmpfn cmpfn, - void *cmparg, - void *temp); - -#define IS_POWER_OF_2(n) ((n) > 0 && ((n) & ((n)-1)) == 0) - -#ifndef MAX -# define MAX(a, b) ((a) > (b) ? (a) : (b)) -#endif - -#ifndef MIN -# define MIN(a, b) ((a) < (b) ? (a) : (b)) -#endif - -/* - * Linux understands that you cannot continue after a failed assert already, - * so we do not need a workaround for platform_assert in linux - */ -__attribute__((noreturn)) void -platform_assert_false(const char *filename, - int linenumber, - const char *functionname, - const char *expr, - const char *message, - ...); - -void -platform_assert_msg(platform_log_handle *log_handle, - const char *filename, - int linenumber, - const char *functionname, - const char *expr, - const char *message, - va_list args); - -/* - * Caller-macro to invoke assertion checking. Avoids a function call for - * most cases when the assertion will succeed. - * - * Note: The dangling fprintf() is really dead-code, as it executes after the - * "noreturn" function implementing the assertion check executes, and fails. - * -BUT- The fprintf() is solely there as a small compile-time check to ensure - * that the arguments match the print-formats in any user-supplied message. - */ -#define platform_assert(expr, ...) \ - ((expr) ? (void)0 \ - : (platform_assert_false( \ - __FILE__, __LINE__, __func__, #expr, "" __VA_ARGS__), \ - (void)fprintf(stderr, " " __VA_ARGS__))) - -static inline timestamp -platform_get_timestamp(void); - -static inline timestamp -platform_timestamp_elapsed(timestamp tv); - -static inline timestamp -platform_get_real_time(void); - -static inline void -platform_sleep_ns(uint64 ns); - -static inline void -platform_semaphore_destroy(platform_semaphore *sema); - -static inline void -platform_semaphore_init(platform_semaphore *sema, - int value, - platform_heap_id heap_id); - -static inline void -platform_semaphore_post(platform_semaphore *sema); - -static inline void -platform_semaphore_wait(platform_semaphore *sema); - -platform_status -platform_histo_create(platform_heap_id heap_id, - uint32 num_buckets, - const int64 *const bucket_limits, - platform_histo_handle *histo); - -void -platform_histo_destroy(platform_heap_id heap_id, platform_histo_handle *histo); - -void -platform_histo_print(platform_histo_handle histo, - const char *name, - platform_log_handle *log_handle); - -static inline threadid -platform_get_tid(); - -static inline void -platform_set_tid(threadid t); - -static inline size_t -platform_strnlen(const char *s, size_t maxlen); - -platform_log_handle * -platform_get_stdout_stream(void); - -typedef struct shmem_heap shmem_heap; - -platform_status -platform_heap_create(platform_module_id module_id, - size_t max, - bool use_shmem, - platform_heap_id *heap_id); - -void -platform_heap_destroy(platform_heap_id *heap_id); - -void -platform_shm_set_splinterdb_handle(platform_heap_id heap_id, void *addr); - -shmem_heap * -platform_heap_id_to_shmaddr(platform_heap_id hid); - -platform_status -platform_buffer_init(buffer_handle *bh, size_t length); - -void * -platform_buffer_getaddr(const buffer_handle *bh); - -platform_status -platform_buffer_deinit(buffer_handle *bh); - -platform_status -platform_mutex_init(platform_mutex *mu, - platform_module_id module_id, - platform_heap_id heap_id); - -platform_status -platform_mutex_destroy(platform_mutex *mu); - -platform_status -platform_spinlock_init(platform_spinlock *lock, - platform_module_id module_id, - platform_heap_id heap_id); - -platform_status -platform_spinlock_destroy(platform_spinlock *lock); - -platform_status -platform_thread_create(platform_thread *thread, - bool32 detached, - platform_thread_worker worker, - void *arg, - platform_heap_id heap_id); - -platform_status -platform_thread_join(platform_thread thread); - - -platform_thread -platform_thread_id_self(); - -char * -platform_strtok_r(char *str, const char *delim, platform_strtok_ctx *ctx); - -/* - * Section 5: - * Platform-specific inline implementations - * - * Non-inline implementations belong in a .c file in the platform_* directory. - * Declarations for the non-inline functions can go in platform_inline.h - */ -#include - - -/* - * Section 6: - * Non-platform-specific inline implementations - */ - - -/* - * Similar to the TYPED_MALLOC functions, for all the free functions we need to - * call platform_get_heap_id() from a macro instead of an inline function - * (which may or may not end up inlined) - * Wrap free and free_volatile: - */ -#define platform_free(id, p) \ - do { \ - platform_free_from_heap( \ - id, (p), STRINGIFY(p), __func__, __FILE__, __LINE__); \ - (p) = NULL; \ - } while (0) - -#define platform_free_volatile(id, p) \ - do { \ - platform_free_volatile_from_heap( \ - id, (p), STRINGIFY(p), __func__, __FILE__, __LINE__); \ - (p) = NULL; \ - } while (0) - -// Convenience function to free something volatile -static inline void -platform_free_volatile_from_heap(platform_heap_id heap_id, - volatile void *ptr, - const char *objname, - const char *func, - const char *file, - int lineno) -{ - // Ok to discard volatile qualifier for free - platform_free_from_heap(heap_id, (void *)ptr, objname, func, file, lineno); -} - -static inline void * -platform_aligned_zalloc(platform_heap_id heap_id, - size_t alignment, - size_t size, - const char *objname, - const char *func, - const char *file, - int lineno) -{ - void *x = platform_aligned_malloc( - heap_id, alignment, size, objname, func, file, lineno); - if (LIKELY(x)) { - memset(x, 0, size); - } - return x; -} - -static inline size_t -max_size_t(size_t a, size_t b) -{ - return a > b ? a : b; -} - -static inline bool32 -SUCCESS(const platform_status s) -{ - return STATUS_IS_EQ(s, STATUS_OK); -} - -platform_status -platform_condvar_init(platform_condvar *cv, platform_heap_id heap_id); - -platform_status -platform_condvar_wait(platform_condvar *cv); - -platform_status -platform_condvar_signal(platform_condvar *cv); - -platform_status -platform_condvar_broadcast(platform_condvar *cv); - -/* calculate difference between two pointers */ -static inline ptrdiff_t -diff_ptr(const void *base, const void *limit) -{ - _Static_assert(sizeof(char) == 1, "Assumption violated"); - return (char *)limit - (char *)base; -} - -#define DEFAULT_THROTTLE_INTERVAL_SEC (60) - -static inline int -platform_backtrace(void **buffer, int size) -{ - return backtrace(buffer, size); -} - -#endif // PLATFORM_H +#include "splinterdb/platform_linux/public_platform.h" +#include "platform_assert.h" +#include "platform_util.h" +#include "platform_machine.h" +#include "platform_status.h" +#include "platform_log.h" +#include "platform_heap.h" +#include "platform_io.h" +#include "platform_sort.h" +#include "platform_string.h" +#include "platform_io.h" +#include "platform_buffer.h" +#include "platform_condvar.h" +#include "platform_mutex.h" +#include "platform_threads.h" +#include "platform_time.h" +#include "platform_util.h" +#include "platform_log.h" +#include "platform_status.h" +#include "platform_typed_alloc.h" +#include "platform_sleep.h" +#include "platform_hash.h" +#include "platform_spinlock.h" \ No newline at end of file diff --git a/src/platform_linux/platform_assert.c b/src/platform_linux/platform_assert.c new file mode 100644 index 000000000..c5006cc3c --- /dev/null +++ b/src/platform_linux/platform_assert.c @@ -0,0 +1,71 @@ +// Copyright 2018-2026 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +#include "platform_assert.h" +#include "platform_log.h" +#include "platform_threads.h" +#include +#include +#include +#include +#include +#include + +/* + * platform_assert_false() - + * + * Platform-specific assert implementation, with support to print an optional + * message and arguments involved in the assertion failure. The caller-macro + * ensures that this function will only be called when the 'exprval' is FALSE; + * i.e. the assertion is failing. + */ +__attribute__((noreturn)) void +platform_assert_false(const char *filename, + int linenumber, + const char *functionname, + const char *expr, + const char *message, + ...) +{ + va_list varargs; + va_start(varargs, message); + + // Run-time assertion messages go to stderr. + platform_assert_msg(Platform_error_log_handle, + filename, + linenumber, + functionname, + expr, + message, + varargs); + va_end(varargs); + platform_error_log("\n"); + + abort(); +} + +/* + * Lower-level function to generate the assertion message, alone. + */ +void +platform_assert_msg(platform_log_handle *log_handle, + const char *filename, + int linenumber, + const char *functionname, + const char *expr, + const char *message, + va_list varargs) +{ + static char assert_msg_fmt[] = "OS-pid=%d, OS-tid=%lu, Thread-ID=%lu, " + "Assertion failed at %s:%d:%s(): \"%s\". "; + platform_log(log_handle, + assert_msg_fmt, + platform_get_os_pid(), + gettid(), + platform_get_tid(), + filename, + linenumber, + functionname, + expr); + vfprintf(log_handle, message, varargs); +} diff --git a/src/platform_linux/platform_assert.h b/src/platform_linux/platform_assert.h new file mode 100644 index 000000000..971a2e11f --- /dev/null +++ b/src/platform_linux/platform_assert.h @@ -0,0 +1,72 @@ +// Copyright 2018-2026 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "splinterdb/platform_linux/public_platform.h" +#include +#include + +#if !defined(SPLINTER_DEBUG) +# define SPLINTER_DEBUG 0 +#else +# if SPLINTER_DEBUG != 0 && SPLINTER_DEBUG != 1 +# error SPLINTER_DEBUG not 0 or 1 +# endif +#endif + +/* + * Linux understands that you cannot continue after a failed assert already, + * so we do not need a workaround for platform_assert in linux + */ +__attribute__((noreturn)) void +platform_assert_false(const char *filename, + int linenumber, + const char *functionname, + const char *expr, + const char *message, + ...); + +void +platform_assert_msg(platform_log_handle *log_handle, + const char *filename, + int linenumber, + const char *functionname, + const char *expr, + const char *message, + va_list args); + +/* + * Caller-macro to invoke assertion checking. Avoids a function call for + * most cases when the assertion will succeed. + * + * Note: The dangling fprintf() is really dead-code, as it executes after the + * "noreturn" function implementing the assertion check executes, and fails. + * -BUT- The fprintf() is solely there as a small compile-time check to ensure + * that the arguments match the print-formats in any user-supplied message. + */ +#define platform_assert(expr, ...) \ + ((expr) ? (void)0 \ + : (platform_assert_false( \ + __FILE__, __LINE__, __func__, #expr, "" __VA_ARGS__), \ + (void)fprintf(stderr, " " __VA_ARGS__))) + +/* + * Section 4: + * Shared function declarations. + * Shared inline function declarations (including declarations for the + * platform-specific inline functions). + * Implementations of shared inline functions that aren't platform specific + * (These may be candidates to move outside of platform.h entirely) + */ +#if SPLINTER_DEBUG +# define debug_assert(expr, ...) platform_assert(expr, __VA_ARGS__) +# define debug_only __attribute__((__unused__)) +# define debug_code(...) __VA_ARGS__ +#else +# define debug_assert(expr, ...) +# define debug_only __attribute__((__unused__)) +# define debug_code(...) +#endif // SPLINTER_DEBUG + +#define platform_assert_status_ok(_s) platform_assert(SUCCESS(_s)); diff --git a/src/platform_linux/platform_backtrace.h b/src/platform_linux/platform_backtrace.h new file mode 100644 index 000000000..757cd3ad0 --- /dev/null +++ b/src/platform_linux/platform_backtrace.h @@ -0,0 +1,11 @@ +// Copyright 2018-2026 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once +#include + +static inline int +platform_backtrace(void **buffer, int size) +{ + return backtrace(buffer, size); +} diff --git a/src/platform_linux/platform_buffer.c b/src/platform_linux/platform_buffer.c new file mode 100644 index 000000000..fe87f69ad --- /dev/null +++ b/src/platform_linux/platform_buffer.c @@ -0,0 +1,94 @@ +// Copyright 2018-2026 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +#include "platform_buffer.h" +#include "platform_log.h" +#include + +bool32 platform_use_hugetlb = FALSE; +bool32 platform_use_mlock = FALSE; + +/* + * Certain modules, e.g. the buffer cache, need a very large buffer which + * may not be serviceable by the heap. Create the requested buffer using + * mmap() and initialize the input 'bh' to track this memory allocation. + */ +platform_status +platform_buffer_init(buffer_handle *bh, size_t length) +{ + platform_status rc = STATUS_NO_MEMORY; + + int prot = PROT_READ | PROT_WRITE; + + // Technically, for threaded execution model, MAP_PRIVATE is sufficient. + // And we only need to create this mmap()'ed buffer in MAP_SHARED for + // process-execution mode. But, at this stage, we don't know apriori if + // we will be using SplinterDB in a multi-process execution environment. + // So, always create this in SHARED mode. This still works for multiple + // threads. + int flags = MAP_SHARED | MAP_ANONYMOUS | MAP_NORESERVE; + if (platform_use_hugetlb) { + flags |= MAP_HUGETLB; + } + + bh->addr = mmap(NULL, length, prot, flags, -1, 0); + if (bh->addr == MAP_FAILED) { + platform_error_log( + "mmap (%lu bytes) failed with error: %s\n", length, strerror(errno)); + goto error; + } + + if (platform_use_mlock) { + int mlock_rv = mlock(bh->addr, length); + if (mlock_rv != 0) { + platform_error_log("mlock (%lu bytes) failed with error: %s\n", + length, + strerror(errno)); + // Save off rc from mlock()-failure, so we return that as status + rc = CONST_STATUS(errno); + + int rv = munmap(bh->addr, length); + if (rv != 0) { + // We are losing rc from this failure; can't return 2 statuses + platform_error_log("munmap %p (%lu bytes) failed with error: %s\n", + bh->addr, + length, + strerror(errno)); + } + goto error; + } + } + bh->length = length; + return STATUS_OK; + +error: + // Reset, in case mmap() or mlock() failed. + bh->addr = NULL; + return rc; +} + +void * +platform_buffer_getaddr(const buffer_handle *bh) +{ + return bh->addr; +} + +/* + * platform_buffer_deinit() - Deinit the buffer handle, which involves + * unmapping the memory for the large buffer created using mmap(). + * This is expected to be called on a 'bh' that has been successfully + * initialized, thru a prior platform_buffer_init() call. + */ +platform_status +platform_buffer_deinit(buffer_handle *bh) +{ + int ret; + ret = munmap(bh->addr, bh->length); + if (ret) { + return CONST_STATUS(errno); + } + + bh->addr = NULL; + bh->length = 0; + return STATUS_OK; +} diff --git a/src/platform_linux/platform_buffer.h b/src/platform_linux/platform_buffer.h new file mode 100644 index 000000000..e84da241d --- /dev/null +++ b/src/platform_linux/platform_buffer.h @@ -0,0 +1,25 @@ +// Copyright 2018-2026 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "splinterdb/platform_linux/public_platform.h" +#include "platform_status.h" + +extern bool32 platform_use_hugetlb; +extern bool32 platform_use_mlock; + +// Buffer handle +typedef struct { + void *addr; + size_t length; +} buffer_handle; + +platform_status +platform_buffer_init(buffer_handle *bh, size_t length); + +void * +platform_buffer_getaddr(const buffer_handle *bh); + +platform_status +platform_buffer_deinit(buffer_handle *bh); diff --git a/src/platform_linux/platform_condvar.c b/src/platform_linux/platform_condvar.c new file mode 100644 index 000000000..e9410b7a8 --- /dev/null +++ b/src/platform_linux/platform_condvar.c @@ -0,0 +1,106 @@ +// Copyright 2018-2026 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +#include "platform_condvar.h" +#include "platform_assert.h" +#include "platform_status.h" +#include "platform_log.h" +#include + +platform_status +platform_condvar_init(platform_condvar *cv, platform_heap_id heap_id) +{ + platform_status status; + bool32 pth_condattr_setpshared_failed = FALSE; + + // Init mutex so it can be shared between processes, if so configured + pthread_mutexattr_t mattr; + pthread_mutexattr_init(&mattr); + status.r = pthread_mutexattr_setpshared(&mattr, PTHREAD_PROCESS_SHARED); + if (!SUCCESS(status)) { + goto mutexattr_setpshared_failed; + } + + // clang-format off + status.r = pthread_mutex_init(&cv->lock, + ((heap_id == PROCESS_PRIVATE_HEAP_ID) + ? NULL : &mattr)); + // clang-format on + debug_only int rv = pthread_mutexattr_destroy(&mattr); + debug_assert(rv == 0); + + if (!SUCCESS(status)) { + goto mutex_init_failed; + } + + // Init condition so it can be shared between processes, if so configured + pthread_condattr_t cattr; + pthread_condattr_init(&cattr); + status.r = pthread_condattr_setpshared(&cattr, PTHREAD_PROCESS_SHARED); + if (!SUCCESS(status)) { + pth_condattr_setpshared_failed = TRUE; + goto condattr_setpshared_failed; + } + + // clang-format off + status.r = pthread_cond_init(&cv->cond, + ((heap_id == PROCESS_PRIVATE_HEAP_ID) + ? NULL : &cattr)); + // clang-format on + + rv = pthread_condattr_destroy(&cattr); + debug_assert(rv == 0); + + // Upon a failure, before exiting, release mutex init'ed above ... + if (!SUCCESS(status)) { + goto cond_init_failed; + } + + return status; + + int ret = 0; +cond_init_failed: +condattr_setpshared_failed: + ret = pthread_mutex_destroy(&cv->lock); + // Yikes! Even this failed. We will lose the prev errno ... + if (ret) { + platform_error_log("%s() failed with error: %s\n", + (pth_condattr_setpshared_failed + ? "pthread_condattr_setpshared" + : "pthread_cond_init"), + platform_status_to_string(status)); + + // Return most recent failure rc + return CONST_STATUS(ret); + } +mutex_init_failed: +mutexattr_setpshared_failed: + return status; +} + +platform_status +platform_condvar_wait(platform_condvar *cv) +{ + int status; + + status = pthread_cond_wait(&cv->cond, &cv->lock); + return CONST_STATUS(status); +} + +platform_status +platform_condvar_signal(platform_condvar *cv) +{ + int status; + + status = pthread_cond_signal(&cv->cond); + return CONST_STATUS(status); +} + +platform_status +platform_condvar_broadcast(platform_condvar *cv) +{ + int status; + + status = pthread_cond_broadcast(&cv->cond); + return CONST_STATUS(status); +} diff --git a/src/platform_linux/platform_condvar.h b/src/platform_linux/platform_condvar.h new file mode 100644 index 000000000..da8376b6e --- /dev/null +++ b/src/platform_linux/platform_condvar.h @@ -0,0 +1,51 @@ +// Copyright 2018-2026 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "platform_status.h" +#include "platform_heap.h" +#include + +typedef struct platform_condvar { + pthread_mutex_t lock; + pthread_cond_t cond; +} platform_condvar; + +platform_status +platform_condvar_init(platform_condvar *cv, platform_heap_id heap_id); + +platform_status +platform_condvar_wait(platform_condvar *cv); + +platform_status +platform_condvar_signal(platform_condvar *cv); + +platform_status +platform_condvar_broadcast(platform_condvar *cv); + + +static inline platform_status +platform_condvar_lock(platform_condvar *cv) +{ + int status; + + status = pthread_mutex_lock(&cv->lock); + return CONST_STATUS(status); +} + +static inline platform_status +platform_condvar_unlock(platform_condvar *cv) +{ + int status; + + status = pthread_mutex_unlock(&cv->lock); + return CONST_STATUS(status); +} + +static inline void +platform_condvar_destroy(platform_condvar *cv) +{ + pthread_mutex_destroy(&cv->lock); + pthread_cond_destroy(&cv->cond); +} diff --git a/src/platform_linux/platform_hash.h b/src/platform_linux/platform_hash.h new file mode 100644 index 000000000..8d4a099e0 --- /dev/null +++ b/src/platform_linux/platform_hash.h @@ -0,0 +1,31 @@ +// Copyright 2018-2026 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "splinterdb/platform_linux/public_platform.h" +#include + +// hash functions +typedef uint32 (*hash_fn)(const void *input, size_t length, unsigned int seed); + +#define HASH_SEED (42) + +// checksums +typedef XXH32_hash_t checksum32; +typedef XXH64_hash_t checksum64; +typedef XXH128_hash_t checksum128; + +#define platform_checksum32 XXH32 +#define platform_checksum64 XXH64 +#define platform_checksum128 XXH128 + +#define platform_hash32 XXH32 +#define platform_hash64 XXH64 +#define platform_hash128 XXH128 + +static inline bool32 +platform_checksum_is_equal(checksum128 left, checksum128 right) +{ + return XXH128_isEqual(left, right); +} diff --git a/src/platform_linux/platform_heap.c b/src/platform_linux/platform_heap.c new file mode 100644 index 000000000..654bafbff --- /dev/null +++ b/src/platform_linux/platform_heap.c @@ -0,0 +1,48 @@ +// Copyright 2018-2026 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +#include "platform_heap.h" +#include "platform_status.h" + +/* + * Declare globals to track heap handle/ID that may have been created when + * using shared memory. We stash away these handles so that we can return the + * right handle via platform_get_heap_id() interface, in case shared segments + * are in use. + */ +platform_heap_id Heap_id = NULL; + +/* + * platform_heap_create() - Create a heap for memory allocation. + * + * By default, we just revert to process' heap-memory and use malloc() / free() + * for memory management. If Splinter is run with shared-memory configuration, + * create a shared-segment which acts as the 'heap' for memory allocation. + */ +platform_status +platform_heap_create(platform_module_id UNUSED_PARAM(module_id), + size_t max, + bool use_shmem, + platform_heap_id *heap_id) +{ + *heap_id = PROCESS_PRIVATE_HEAP_ID; + + if (use_shmem) { + platform_status rc = platform_shmcreate(max, (shmem_heap **)heap_id); + if (SUCCESS(rc)) { + Heap_id = *heap_id; + } + return rc; + } + *heap_id = NULL; + return STATUS_OK; +} + +void +platform_heap_destroy(platform_heap_id *heap_id) +{ + // If shared segment was allocated, it's being tracked thru heap ID. + if (*heap_id) { + return platform_shmdestroy((shmem_heap **)heap_id); + } +} diff --git a/src/platform_linux/platform_heap.h b/src/platform_linux/platform_heap.h new file mode 100644 index 000000000..3c39256b1 --- /dev/null +++ b/src/platform_linux/platform_heap.h @@ -0,0 +1,212 @@ +// Copyright 2018-2026 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "platform_assert.h" +#include "platform_status.h" +#include "platform_util.h" +#include "platform_machine.h" +#include "shmem.h" +#include +#include + +typedef void *platform_heap_id; + +/* Don't have a good place for this, so putting it here */ +typedef void *platform_module_id; + +static inline platform_module_id +platform_get_module_id() +{ + // void* NULL since we don't actually need a module id + return NULL; +} + + +extern platform_heap_id Heap_id; + +/* + * Provide a tag for callers that do not want to use shared-memory allocation, + * when configured but want to fallback to default scheme of allocating + * process-private memory. Typically, this would default to malloc()/free(). + * (Clients that repeatedly allocate and free a large chunk of memory in some + * code path would want to use this tag.) + */ +#define PROCESS_PRIVATE_HEAP_ID (platform_heap_id) NULL + + +static inline platform_heap_id +platform_get_heap_id(void) +{ + // void* NULL since we don't actually need a heap id + return Heap_id; +} + +/* + * Return # of bytes needed to align requested 'size' bytes at 'alignment' + * boundary. + */ +static inline size_t +platform_align_bytes_reqd(const size_t alignment, const size_t size) +{ + return ((alignment - (size % alignment)) % alignment); +} + +/* + * platform_aligned_malloc() -- Allocate n-bytes accounting for alignment. + * + * This interface will, by default, allocate using aligned_alloc(). Currently + * this supports alignments up to a cache-line. + * If Splinter is configured to run with shared memory, we will invoke the + * shmem-allocation function, working off of the (non-NULL) platform_heap_id. + */ +static inline void * +platform_aligned_malloc(const platform_heap_id heap_id, + const size_t alignment, // IN + const size_t size, // IN + const char *objname, + const char *func, + const char *file, + const int lineno) +{ + // Requirement for aligned_alloc + platform_assert(IS_POWER_OF_2(alignment)); + + /* + * aligned_alloc requires size to be a multiple of alignment + * round up to nearest multiple of alignment + * + * Note that since this is inlined, the compiler will turn the constant + * (power of 2) alignment mod operations into bitwise & + */ + // RESOLVE: Delete this padding from caller. Push this down to + // platform_shm_alloc(). + const size_t padding = platform_align_bytes_reqd(alignment, size); + const size_t required = (size + padding); + + void *retptr = + (heap_id + ? platform_shm_alloc(heap_id, required, objname, func, file, lineno) + : aligned_alloc(alignment, required)); + return retptr; +} + +/* + * platform_realloc() - Reallocate 'newsize' bytes and copy over old contents. + * + * This is a wrapper around C-realloc() but farms over to shared-memory + * based realloc, when needed. + * + * The interface is intentional to avoid inadvertently swapping 'oldsize' and + * 'newsize' in the call, if they were to appear next to each other. + * + * Reallocing to size 0 must be equivalent to freeing. + * Reallocing from NULL must be equivalent to allocing. + */ +static inline void * +platform_realloc(const platform_heap_id heap_id, + const size_t oldsize, + void *ptr, // IN + const size_t newsize) // IN +{ + /* FIXME: alignment? */ + + // Farm control off to shared-memory based realloc, if it's configured + if (heap_id) { + // The shmem-based allocator is expecting all memory requests to be of + // aligned sizes, as that's what platform_aligned_malloc() does. So, to + // keep that allocator happy, align this memory request if needed. + // As this is the case of realloc, we assume that it would suffice to + // align at platform's natural cacheline boundary. + const size_t padding = + platform_align_bytes_reqd(PLATFORM_CACHELINE_SIZE, newsize); + const size_t required = (newsize + padding); + return platform_shm_realloc( + heap_id, ptr, oldsize, required, __func__, __FILE__, __LINE__); + } else { + return realloc(ptr, newsize); + } +} + +static inline void +platform_free_from_heap(platform_heap_id heap_id, + void *ptr, + const char *objname, + const char *func, + const char *file, + int lineno) +{ + if (heap_id) { + platform_shm_free(heap_id, ptr, objname, func, file, lineno); + } else { + free(ptr); + } +} + +typedef struct shmem_heap shmem_heap; + +platform_status +platform_heap_create(platform_module_id module_id, + size_t max, + bool use_shmem, + platform_heap_id *heap_id); + +void +platform_heap_destroy(platform_heap_id *heap_id); + +void +platform_shm_set_splinterdb_handle(platform_heap_id heap_id, void *addr); + +shmem_heap * +platform_heap_id_to_shmaddr(platform_heap_id hid); + +/* + * Similar to the TYPED_MALLOC functions, for all the free functions we need to + * call platform_get_heap_id() from a macro instead of an inline function + * (which may or may not end up inlined) + * Wrap free and free_volatile: + */ +#define platform_free(id, p) \ + do { \ + platform_free_from_heap( \ + id, (p), STRINGIFY(p), __func__, __FILE__, __LINE__); \ + (p) = NULL; \ + } while (0) + +#define platform_free_volatile(id, p) \ + do { \ + platform_free_volatile_from_heap( \ + id, (p), STRINGIFY(p), __func__, __FILE__, __LINE__); \ + (p) = NULL; \ + } while (0) + +// Convenience function to free something volatile +static inline void +platform_free_volatile_from_heap(platform_heap_id heap_id, + volatile void *ptr, + const char *objname, + const char *func, + const char *file, + int lineno) +{ + // Ok to discard volatile qualifier for free + platform_free_from_heap(heap_id, (void *)ptr, objname, func, file, lineno); +} + +static inline void * +platform_aligned_zalloc(platform_heap_id heap_id, + size_t alignment, + size_t size, + const char *objname, + const char *func, + const char *file, + int lineno) +{ + void *x = platform_aligned_malloc( + heap_id, alignment, size, objname, func, file, lineno); + if (LIKELY(x)) { + memset(x, 0, size); + } + return x; +} diff --git a/src/platform_linux/platform_inline.h b/src/platform_linux/platform_inline.h deleted file mode 100644 index 46b95ee0a..000000000 --- a/src/platform_linux/platform_inline.h +++ /dev/null @@ -1,551 +0,0 @@ -// Copyright 2018-2021 VMware, Inc. -// SPDX-License-Identifier: Apache-2.0 - -#ifndef PLATFORM_LINUX_INLINE_H -#define PLATFORM_LINUX_INLINE_H - -#include -#include -#include // for memcpy, strerror -#include // for nanosecond sleep api. - -#include "shmem.h" - -static inline size_t -platform_strnlen(const char *s, size_t maxlen) -{ - return strnlen(s, maxlen); -} - -static inline uint32 -platform_popcount(uint32 x) -{ - return __builtin_popcount(x); -} - -#define platform_checksum32 XXH32 -#define platform_checksum64 XXH64 -#define platform_checksum128 XXH128 - -#define platform_hash32 XXH32 -#define platform_hash64 XXH64 -#define platform_hash128 XXH128 - -static inline bool32 -platform_checksum_is_equal(checksum128 left, checksum128 right) -{ - return XXH128_isEqual(left, right); -} - -static void -platform_free_from_heap(platform_heap_id UNUSED_PARAM(heap_id), - void *ptr, - const char *objname, - const char *func, - const char *file, - int lineno); - -static inline timestamp -platform_get_timestamp(void) -{ - struct timespec ts; - clock_gettime(CLOCK_MONOTONIC, &ts); - return SEC_TO_NSEC(ts.tv_sec) + ts.tv_nsec; -} - -static inline timestamp -platform_timestamp_elapsed(timestamp tv) -{ - struct timespec ts; - clock_gettime(CLOCK_MONOTONIC, &ts); - return SEC_TO_NSEC(ts.tv_sec) + ts.tv_nsec - tv; -} - -static inline timestamp -platform_timestamp_diff(timestamp start, timestamp end) -{ - return end - start; -} - -static inline timestamp -platform_get_real_time(void) -{ - struct timespec ts; - clock_gettime(CLOCK_REALTIME, &ts); - return SEC_TO_NSEC(ts.tv_sec) + ts.tv_nsec; -} - -static inline void -platform_pause() -{ -#if defined(__i386__) || defined(__x86_64__) - __builtin_ia32_pause(); -#elif defined(__aarch64__) // ARM64 - // pause + memory fence for x64 and ARM - // https://chromium.googlesource.com/chromium/src/third_party/WebKit/Source/wtf/+/823d62cdecdbd5f161634177e130e5ac01eb7b48/SpinLock.cpp - __asm__ __volatile__("yield"); -#else -# error Unknown CPU arch -#endif -} - -static inline void -platform_sleep_ns(uint64 ns) -{ - if (ns < USEC_TO_NSEC(50)) { - for (uint64 i = 0; i < ns / 5 + 1; i++) { - platform_pause(); - } - } else { - struct timespec res; - res.tv_sec = ns / SEC_TO_NSEC(1); - res.tv_nsec = (ns - (res.tv_sec * SEC_TO_NSEC(1))); - clock_nanosleep(CLOCK_MONOTONIC, 0, &res, NULL); - } -} - -static inline void -platform_semaphore_destroy(platform_semaphore *sema) -{ - __attribute__((unused)) int err = sem_destroy(sema); - debug_assert(!err); -} - -/* - * Ref: https://man7.org/linux/man-pages/man3/sem_init.3.html - * for choice of 'pshared' arg to sem_init(). - */ -static inline void -platform_semaphore_init(platform_semaphore *sema, - int value, - platform_heap_id heap_id) -{ - // If we are running with a shared segment, it's likely that we - // may also fork child processes attaching to Splinter's shmem. - // Then, use 1 => spinlocks are shared across process boundaries. - // Else, use 0 => spinlocks are shared between threads in a process. - __attribute__((unused)) int err = - sem_init(sema, ((heap_id == PROCESS_PRIVATE_HEAP_ID) ? 0 : 1), value); - debug_assert(!err); -} - -static inline void -platform_semaphore_post(platform_semaphore *sema) -{ - __attribute__((unused)) int err = sem_post(sema); - debug_assert(!err); -} - -static inline void -platform_semaphore_wait(platform_semaphore *sema) -{ - __attribute__((unused)) int err = sem_wait(sema); - debug_assert(!err); -} - -/* - * STATUS_OK: if wait succeeded - * STATUS_BUSY: if need to retry - * other: failure - */ -static inline platform_status -platform_semaphore_try_wait(platform_semaphore *sema) -{ - int ret = sem_trywait(sema); - - if (ret == 0) { - return STATUS_OK; - } - if (errno == EAGAIN) { - return STATUS_BUSY; - } - - return CONST_STATUS(errno); -} - -static inline platform_status -platform_mutex_lock(platform_mutex *lock) -{ - int ret = pthread_mutex_lock(&lock->mutex); - platform_assert(lock->owner == INVALID_TID, - "Found an unlocked a mutex with an existing owner:\n" - "lock: %p, tid: %lu, owner: %lu\n", - lock, - platform_get_tid(), - lock->owner); - lock->owner = platform_get_tid(); - return CONST_STATUS(ret); -} - -static inline platform_status -platform_mutex_unlock(platform_mutex *lock) -{ - platform_assert(lock->owner == platform_get_tid(), - "Attempt to unlock a mutex without ownership:\n" - "lock: %p, tid: %lu, owner: %lu\n", - lock, - platform_get_tid(), - lock->owner); - lock->owner = INVALID_TID; - int ret = pthread_mutex_unlock(&lock->mutex); - return CONST_STATUS(ret); -} - -static inline platform_status -platform_spin_lock(platform_spinlock *lock) -{ - int ret; - - ret = pthread_spin_lock(lock); - - return CONST_STATUS(ret); -} - -static inline platform_status -platform_spin_unlock(platform_spinlock *lock) -{ - int ret; - - ret = pthread_spin_unlock(lock); - - return CONST_STATUS(ret); -} - -static inline threadid -platform_get_tid() -{ - extern __thread threadid xxxtid; - return xxxtid; -} - -static inline void -platform_set_tid(threadid t) -{ - extern __thread threadid xxxtid; - xxxtid = t; -} - -static inline int -platform_getpid() -{ - return getpid(); -} - -static inline void -platform_yield() -{ -} - -// platform predicates -static inline bool32 -STATUS_IS_EQ(const platform_status s1, const platform_status s2) -{ - return s1.r == s2.r; -} - -static inline bool32 -STATUS_IS_NE(const platform_status s1, const platform_status s2) -{ - return s1.r != s2.r; -} - -static inline const char * -platform_status_to_string(const platform_status status) -{ - return strerror(status.r); -} - -/* Default output file handles for different logging interfaces */ -#define PLATFORM_CR "\r" - -static inline platform_status -platform_open_log_stream(platform_stream_handle *stream) -{ - ZERO_CONTENTS(stream); - stream->stream = open_memstream(&stream->str, &stream->size); - if (stream->stream == NULL) { - return STATUS_NO_MEMORY; - } - return STATUS_OK; -} - -static inline void -platform_flush_log_stream(platform_stream_handle *stream) -{ - fflush(stream->stream); -} - -static inline void -platform_close_log_stream(platform_stream_handle *stream, - platform_log_handle *log_handle) -{ - fclose(stream->stream); - fputs(stream->str, log_handle); - fflush(log_handle); - platform_free_from_heap( - NULL, stream->str, "stream", __func__, __FILE__, __LINE__); -} - -static inline platform_log_handle * -platform_log_stream_to_log_handle(platform_stream_handle *stream) -{ - return stream->stream; -} - -static inline char * -platform_log_stream_to_string(platform_stream_handle *stream) -{ - platform_flush_log_stream(stream); - return stream->str; -} - -#define platform_log(log_handle, ...) \ - do { \ - fprintf((log_handle), __VA_ARGS__); \ - fflush(log_handle); \ - } while (0) - -#define platform_default_log(...) \ - do { \ - platform_log(Platform_default_log_handle, __VA_ARGS__); \ - } while (0) - -#define platform_error_log(...) \ - do { \ - platform_log(Platform_error_log_handle, __VA_ARGS__); \ - } while (0) - -#define platform_log_stream(stream, ...) \ - do { \ - platform_log_handle *log_handle = \ - platform_log_stream_to_log_handle(stream); \ - platform_log(log_handle, __VA_ARGS__); \ - } while (0) - -#define platform_throttled_log(sec, log_handle, ...) \ - do { \ - platform_log(log_handle, __VA_ARGS__); \ - } while (0) - -#define platform_throttled_default_log(sec, ...) \ - do { \ - platform_default_log(__VA_ARGS__); \ - } while (0) - -#define platform_throttled_error_log(sec, ...) \ - do { \ - platform_error_log(__VA_ARGS__); \ - } while (0) - -#define platform_open_log_file(path, mode) \ - ({ \ - platform_log_handle *lh = fopen(path, mode); \ - platform_assert(lh); \ - lh; \ - }) - -#define platform_close_log_file(log_handle) \ - do { \ - fclose(log_handle); \ - } while (0) - -#define platform_thread_cleanup_push(func, arg) \ - pthread_cleanup_push((func), (arg)) - -#define platform_thread_cleanup_pop(exec) pthread_cleanup_pop((exec)) - -static inline void -platform_histo_insert(platform_histo_handle histo, int64 datum) -{ - int lo = 0, hi = histo->num_buckets - 1; - - while (hi > lo) { - int mid = lo + (hi - lo) / 2; - - if (datum > histo->bucket_limits[mid]) { - lo = mid + 1; - } else { - hi = mid - 1; - } - } - platform_assert(lo < histo->num_buckets); - histo->count[lo]++; - if (histo->num == 0) { - histo->min = histo->max = datum; - } else { - histo->max = MAX(histo->max, datum); - histo->min = MIN(histo->min, datum); - } - histo->total += datum; - histo->num++; -} - -static inline void -platform_histo_merge_in(platform_histo_handle dest_histo, - platform_histo_handle src_histo) -{ - uint32 i; - if (src_histo->num == 0) { - return; - } - - platform_assert(dest_histo->num_buckets == src_histo->num_buckets); - for (i = 0; i < dest_histo->num_buckets - 1; i++) { - platform_assert(dest_histo->bucket_limits[i] - == src_histo->bucket_limits[i]); - } - if (src_histo->min < dest_histo->min || dest_histo->num == 0) { - dest_histo->min = src_histo->min; - } - if (src_histo->max > dest_histo->max || dest_histo->num == 0) { - dest_histo->max = src_histo->max; - } - dest_histo->total += src_histo->total; - dest_histo->num += src_histo->num; - - for (i = 0; i < dest_histo->num_buckets; i++) { - dest_histo->count[i] += src_histo->count[i]; - } -} - -static inline platform_heap_id -platform_get_heap_id(void) -{ - // void* NULL since we don't actually need a heap id - return Heap_id; -} - -static inline platform_module_id -platform_get_module_id() -{ - // void* NULL since we don't actually need a module id - return NULL; -} - -/* - * Return # of bytes needed to align requested 'size' bytes at 'alignment' - * boundary. - */ -static inline size_t -platform_align_bytes_reqd(const size_t alignment, const size_t size) -{ - return ((alignment - (size % alignment)) % alignment); -} - -/* - * platform_aligned_malloc() -- Allocate n-bytes accounting for alignment. - * - * This interface will, by default, allocate using aligned_alloc(). Currently - * this supports alignments up to a cache-line. - * If Splinter is configured to run with shared memory, we will invoke the - * shmem-allocation function, working off of the (non-NULL) platform_heap_id. - */ -static inline void * -platform_aligned_malloc(const platform_heap_id heap_id, - const size_t alignment, // IN - const size_t size, // IN - const char *objname, - const char *func, - const char *file, - const int lineno) -{ - // Requirement for aligned_alloc - platform_assert(IS_POWER_OF_2(alignment)); - - /* - * aligned_alloc requires size to be a multiple of alignment - * round up to nearest multiple of alignment - * - * Note that since this is inlined, the compiler will turn the constant - * (power of 2) alignment mod operations into bitwise & - */ - // RESOLVE: Delete this padding from caller. Push this down to - // platform_shm_alloc(). - const size_t padding = platform_align_bytes_reqd(alignment, size); - const size_t required = (size + padding); - - void *retptr = - (heap_id - ? platform_shm_alloc(heap_id, required, objname, func, file, lineno) - : aligned_alloc(alignment, required)); - return retptr; -} - -/* - * platform_realloc() - Reallocate 'newsize' bytes and copy over old contents. - * - * This is a wrapper around C-realloc() but farms over to shared-memory - * based realloc, when needed. - * - * The interface is intentional to avoid inadvertently swapping 'oldsize' and - * 'newsize' in the call, if they were to appear next to each other. - * - * Reallocing to size 0 must be equivalent to freeing. - * Reallocing from NULL must be equivalent to allocing. - */ -static inline void * -platform_realloc(const platform_heap_id heap_id, - const size_t oldsize, - void *ptr, // IN - const size_t newsize) // IN -{ - /* FIXME: alignment? */ - - // Farm control off to shared-memory based realloc, if it's configured - if (heap_id) { - // The shmem-based allocator is expecting all memory requests to be of - // aligned sizes, as that's what platform_aligned_malloc() does. So, to - // keep that allocator happy, align this memory request if needed. - // As this is the case of realloc, we assume that it would suffice to - // align at platform's natural cacheline boundary. - const size_t padding = - platform_align_bytes_reqd(PLATFORM_CACHELINE_SIZE, newsize); - const size_t required = (newsize + padding); - return platform_shm_realloc( - heap_id, ptr, oldsize, required, __func__, __FILE__, __LINE__); - } else { - return realloc(ptr, newsize); - } -} - -static inline void -platform_free_from_heap(platform_heap_id heap_id, - void *ptr, - const char *objname, - const char *func, - const char *file, - int lineno) -{ - if (heap_id) { - platform_shm_free(heap_id, ptr, objname, func, file, lineno); - } else { - free(ptr); - } -} - -static inline platform_status -platform_condvar_lock(platform_condvar *cv) -{ - int status; - - status = pthread_mutex_lock(&cv->lock); - return CONST_STATUS(status); -} - -static inline platform_status -platform_condvar_unlock(platform_condvar *cv) -{ - int status; - - status = pthread_mutex_unlock(&cv->lock); - return CONST_STATUS(status); -} - -static inline void -platform_condvar_destroy(platform_condvar *cv) -{ - pthread_mutex_destroy(&cv->lock); - pthread_cond_destroy(&cv->cond); -} - -#endif // PLATFORM_LINUX_INLINE_H diff --git a/src/platform_linux/platform_io.c b/src/platform_linux/platform_io.c new file mode 100644 index 000000000..e0da55071 --- /dev/null +++ b/src/platform_linux/platform_io.c @@ -0,0 +1,23 @@ +// Copyright 2018-2026 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +#include "platform_io.h" +#include "laio.h" + +io_handle * +io_handle_create(io_config *cfg, platform_heap_id hid) +{ + return laio_handle_create(cfg, hid); +} + +void +io_handle_destroy(io_handle *ioh) +{ + laio_handle_destroy(ioh); +} + +platform_status +io_config_valid(io_config *cfg) +{ + return laio_config_valid(cfg); +} diff --git a/src/io.h b/src/platform_linux/platform_io.h similarity index 88% rename from src/io.h rename to src/platform_linux/platform_io.h index b9c8dc5a5..1db7cb764 100644 --- a/src/io.h +++ b/src/platform_linux/platform_io.h @@ -1,4 +1,4 @@ -// Copyright 2018-2021 VMware, Inc. +// Copyright 2018-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* @@ -10,12 +10,36 @@ #pragma once #include "async.h" -#include "platform.h" +#include "platform_status.h" +#include "platform_string.h" +#include "platform_util.h" +#include "platform_heap.h" +#include typedef struct io_handle io_handle; typedef struct io_async_req io_async_req; typedef struct io_async_state io_async_state; +struct iovec; + +/* + * SplinterDB can be configured with different page-sizes, given by these + * min & max values. + */ +#define IO_MIN_PAGE_SIZE (4096) +#define IO_MAX_PAGE_SIZE (8192) + +#define IO_DEFAULT_PAGE_SIZE IO_MIN_PAGE_SIZE +#define IO_DEFAULT_PAGES_PER_EXTENT 32 +#define IO_DEFAULT_EXTENT_SIZE \ + (IO_DEFAULT_PAGES_PER_EXTENT * IO_DEFAULT_PAGE_SIZE) + +#define IO_DEFAULT_FLAGS (O_RDWR | O_CREAT) +#define IO_DEFAULT_PERMS (0755) +#define IO_DEFAULT_KERNEL_QUEUE_SIZE (256) +#define IO_DEFAULT_FILENAME "db" +#define IO_DEFAULT_ASYNC_QUEUE_DEPTH (256) + /* * IO Configuration structure - used to setup the run-time IO system. */ @@ -103,12 +127,6 @@ struct io_async_state { const io_async_state_ops *ops; }; -platform_status -io_handle_init(platform_io_handle *ioh, io_config *cfg, platform_heap_id hid); - -void -io_handle_deinit(platform_io_handle *ioh); - static inline platform_status io_read(io_handle *io, void *buf, uint64 bytes, uint64 addr) { @@ -217,6 +235,7 @@ io_max_latency_elapsed(io_handle *io, timestamp ts) * deallocate io_filename once this returns) *----------------------------------------------------------------------------- */ + static inline void io_config_init(io_config *io_cfg, uint64 page_size, @@ -238,3 +257,13 @@ io_config_init(io_config *io_cfg, io_cfg->perms = perms; io_cfg->kernel_queue_size = async_queue_depth; } + +platform_status +io_config_valid(io_config *cfg); + + +io_handle * +io_handle_create(io_config *cfg, platform_heap_id hid); + +void +io_handle_destroy(io_handle *ioh); diff --git a/src/platform_linux/platform_log.c b/src/platform_linux/platform_log.c new file mode 100644 index 000000000..7df73609f --- /dev/null +++ b/src/platform_linux/platform_log.c @@ -0,0 +1,41 @@ +// Copyright 2018-2026 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +#include "platform_log.h" +#include "platform_assert.h" + +// By default, platform_default_log() messages are sent to /dev/null +// and platform_error_log() messages go to stderr (see below). +// +// Use platform_set_log_streams() to send the log messages elsewhere. +platform_log_handle *Platform_default_log_handle = NULL; +platform_log_handle *Platform_error_log_handle = NULL; + +// This function is run automatically at library-load time +void __attribute__((constructor)) +platform_init_log_file_handles(void) +{ + FILE *dev_null_file = fopen("/dev/null", "w"); + platform_assert(dev_null_file != NULL); + + Platform_default_log_handle = dev_null_file; + Platform_error_log_handle = stderr; +} + +// Set the streams where informational and error messages will be printed. +void +platform_set_log_streams(platform_log_handle *info_stream, + platform_log_handle *error_stream) +{ + platform_assert(info_stream != NULL); + platform_assert(error_stream != NULL); + Platform_default_log_handle = info_stream; + Platform_error_log_handle = error_stream; +} + +// Return the stdout log-stream handle +platform_log_handle * +platform_get_stdout_stream(void) +{ + return Platform_default_log_handle; +} diff --git a/src/platform_linux/platform_log.h b/src/platform_linux/platform_log.h new file mode 100644 index 000000000..92433756b --- /dev/null +++ b/src/platform_linux/platform_log.h @@ -0,0 +1,125 @@ +// Copyright 2018-2026 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "splinterdb/platform_linux/public_platform.h" +#include "platform_status.h" +#include "platform_util.h" +#include "platform_heap.h" + +/* Default output file handles for different logging interfaces */ +#define PLATFORM_CR "\r" + +#define DEFAULT_THROTTLE_INTERVAL_SEC (60) + + +/* + * A handle which buffers streamed content to be atomically written to a + * platform_log_handle. + */ +typedef struct { + char *str; + size_t size; + FILE *stream; +} platform_stream_handle; + +extern platform_log_handle *Platform_default_log_handle; +extern platform_log_handle *Platform_error_log_handle; + +platform_log_handle * +platform_get_stdout_stream(void); + +static inline platform_status +platform_open_log_stream(platform_stream_handle *stream) +{ + ZERO_CONTENTS(stream); + stream->stream = open_memstream(&stream->str, &stream->size); + if (stream->stream == NULL) { + return STATUS_NO_MEMORY; + } + return STATUS_OK; +} + +static inline void +platform_flush_log_stream(platform_stream_handle *stream) +{ + fflush(stream->stream); +} + +static inline void +platform_close_log_stream(platform_stream_handle *stream, + platform_log_handle *log_handle) +{ + fclose(stream->stream); + fputs(stream->str, log_handle); + fflush(log_handle); + platform_free_from_heap( + NULL, stream->str, "stream", __func__, __FILE__, __LINE__); +} + +static inline platform_log_handle * +platform_log_stream_to_log_handle(platform_stream_handle *stream) +{ + return stream->stream; +} + +static inline char * +platform_log_stream_to_string(platform_stream_handle *stream) +{ + platform_flush_log_stream(stream); + return stream->str; +} + +#define platform_log(log_handle, ...) \ + do { \ + fprintf((log_handle), __VA_ARGS__); \ + fflush(log_handle); \ + } while (0) + +#define platform_default_log(...) \ + do { \ + platform_log(Platform_default_log_handle, __VA_ARGS__); \ + } while (0) + +#define platform_error_log(...) \ + do { \ + platform_log(Platform_error_log_handle, __VA_ARGS__); \ + } while (0) + +#define platform_log_stream(stream, ...) \ + do { \ + platform_log_handle *log_handle = \ + platform_log_stream_to_log_handle(stream); \ + platform_log(log_handle, __VA_ARGS__); \ + } while (0) + +#define platform_throttled_log(sec, log_handle, ...) \ + do { \ + platform_log(log_handle, __VA_ARGS__); \ + } while (0) + +#define platform_throttled_default_log(sec, ...) \ + do { \ + platform_default_log(__VA_ARGS__); \ + } while (0) + +#define platform_throttled_error_log(sec, ...) \ + do { \ + platform_error_log(__VA_ARGS__); \ + } while (0) + +#define platform_open_log_file(path, mode) \ + ({ \ + platform_log_handle *lh = fopen(path, mode); \ + platform_assert(lh); \ + lh; \ + }) + +#define platform_close_log_file(log_handle) \ + do { \ + fclose(log_handle); \ + } while (0) + +#define FRACTION_FMT(w, s) "%" STRINGIFY_VALUE(w) "." STRINGIFY_VALUE(s) "f" +#define FRACTION_ARGS(f) ((double)(f).numerator / (double)(f).denominator) diff --git a/src/platform_linux/platform_machine.h b/src/platform_linux/platform_machine.h new file mode 100644 index 000000000..876bc436c --- /dev/null +++ b/src/platform_linux/platform_machine.h @@ -0,0 +1,18 @@ +// Copyright 2018-2026 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "splinterdb/platform_linux/public_platform.h" + +#define PLATFORM_CACHELINE_SIZE 64 +#define PLATFORM_CACHELINE_ALIGNED \ + __attribute__((__aligned__(PLATFORM_CACHELINE_SIZE))) + +typedef struct { + uint32 v; +} PLATFORM_CACHELINE_ALIGNED cache_aligned_uint32; + + +_Static_assert(sizeof(cache_aligned_uint32) == PLATFORM_CACHELINE_SIZE, + "Attribute set wrong"); diff --git a/src/platform_linux/platform_mutex.c b/src/platform_linux/platform_mutex.c new file mode 100644 index 000000000..2527e5ff3 --- /dev/null +++ b/src/platform_linux/platform_mutex.c @@ -0,0 +1,43 @@ +// Copyright 2018-2026 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +#include "platform_mutex.h" +#include "platform_assert.h" +#include "platform_status.h" +#include + +platform_status +platform_mutex_init(platform_mutex *lock, + platform_module_id UNUSED_PARAM(module_id), + platform_heap_id heap_id) +{ + platform_status status; + // Init mutex so it can be shared between processes, if so configured + pthread_mutexattr_t mattr; + pthread_mutexattr_init(&mattr); + status.r = pthread_mutexattr_setpshared(&mattr, PTHREAD_PROCESS_SHARED); + if (!SUCCESS(status)) { + return status; + } + + // clang-format off + status.r = pthread_mutex_init(&lock->mutex, + ((heap_id == PROCESS_PRIVATE_HEAP_ID) + ? NULL : &mattr)); + // clang-format on + + // Mess with output vars only in case of a success + if (SUCCESS(status)) { + lock->owner = INVALID_TID; + } + return status; +} + +platform_status +platform_mutex_destroy(platform_mutex *lock) +{ + // Cannot call destroy on a locked lock + platform_assert(lock->owner == INVALID_TID); + int ret = pthread_mutex_destroy(&lock->mutex); + return CONST_STATUS(ret); +} diff --git a/src/platform_linux/platform_mutex.h b/src/platform_linux/platform_mutex.h new file mode 100644 index 000000000..a043a5e44 --- /dev/null +++ b/src/platform_linux/platform_mutex.h @@ -0,0 +1,51 @@ +// Copyright 2018-2026 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "splinterdb/platform_linux/public_platform.h" +#include "platform_assert.h" +#include "platform_threads.h" +#include "platform_status.h" + +// Thread-specific mutex, with ownership tracking. +typedef struct platform_mutex { + pthread_mutex_t mutex; + threadid owner; +} platform_mutex; + +platform_status +platform_mutex_init(platform_mutex *mu, + platform_module_id module_id, + platform_heap_id heap_id); + +platform_status +platform_mutex_destroy(platform_mutex *mu); + +static inline platform_status +platform_mutex_lock(platform_mutex *lock) +{ + int ret = pthread_mutex_lock(&lock->mutex); + platform_assert(lock->owner == INVALID_TID, + "Found an unlocked a mutex with an existing owner:\n" + "lock: %p, tid: %lu, owner: %lu\n", + lock, + platform_get_tid(), + lock->owner); + lock->owner = platform_get_tid(); + return CONST_STATUS(ret); +} + +static inline platform_status +platform_mutex_unlock(platform_mutex *lock) +{ + platform_assert(lock->owner == platform_get_tid(), + "Attempt to unlock a mutex without ownership:\n" + "lock: %p, tid: %lu, owner: %lu\n", + lock, + platform_get_tid(), + lock->owner); + lock->owner = INVALID_TID; + int ret = pthread_mutex_unlock(&lock->mutex); + return CONST_STATUS(ret); +} diff --git a/src/platform_linux/platform_semaphore.h b/src/platform_linux/platform_semaphore.h new file mode 100644 index 000000000..5cf7c221e --- /dev/null +++ b/src/platform_linux/platform_semaphore.h @@ -0,0 +1,68 @@ +// Copyright 2018-2026 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once +#include "platform_heap.h" +#include + + +typedef sem_t platform_semaphore; + +static inline void +platform_semaphore_destroy(platform_semaphore *sema) +{ + __attribute__((unused)) int err = sem_destroy(sema); + debug_assert(!err); +} + +/* + * Ref: https://man7.org/linux/man-pages/man3/sem_init.3.html + * for choice of 'pshared' arg to sem_init(). + */ +static inline void +platform_semaphore_init(platform_semaphore *sema, + int value, + platform_heap_id heap_id) +{ + // If we are running with a shared segment, it's likely that we + // may also fork child processes attaching to Splinter's shmem. + // Then, use 1 => spinlocks are shared across process boundaries. + // Else, use 0 => spinlocks are shared between threads in a process. + __attribute__((unused)) int err = + sem_init(sema, ((heap_id == PROCESS_PRIVATE_HEAP_ID) ? 0 : 1), value); + debug_assert(!err); +} + +static inline void +platform_semaphore_post(platform_semaphore *sema) +{ + __attribute__((unused)) int err = sem_post(sema); + debug_assert(!err); +} + +static inline void +platform_semaphore_wait(platform_semaphore *sema) +{ + __attribute__((unused)) int err = sem_wait(sema); + debug_assert(!err); +} + +/* + * STATUS_OK: if wait succeeded + * STATUS_BUSY: if need to retry + * other: failure + */ +static inline platform_status +platform_semaphore_try_wait(platform_semaphore *sema) +{ + int ret = sem_trywait(sema); + + if (ret == 0) { + return STATUS_OK; + } + if (errno == EAGAIN) { + return STATUS_BUSY; + } + + return CONST_STATUS(errno); +} diff --git a/src/platform_linux/platform_sleep.h b/src/platform_linux/platform_sleep.h new file mode 100644 index 000000000..9b780dc38 --- /dev/null +++ b/src/platform_linux/platform_sleep.h @@ -0,0 +1,42 @@ +// Copyright 2018-2026 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "splinterdb/platform_linux/public_platform.h" +#include "platform_units.h" +#include + +static inline void +platform_pause() +{ +#if defined(__i386__) || defined(__x86_64__) + __builtin_ia32_pause(); +#elif defined(__aarch64__) // ARM64 + // pause + memory fence for x64 and ARM + // https://chromium.googlesource.com/chromium/src/third_party/WebKit/Source/wtf/+/823d62cdecdbd5f161634177e130e5ac01eb7b48/SpinLock.cpp + __asm__ __volatile__("yield"); +#else +# error Unknown CPU arch +#endif +} + +static inline void +platform_sleep_ns(uint64 ns) +{ + if (ns < USEC_TO_NSEC(50)) { + for (uint64 i = 0; i < ns / 5 + 1; i++) { + platform_pause(); + } + } else { + struct timespec res; + res.tv_sec = ns / SEC_TO_NSEC(1); + res.tv_nsec = (ns - (res.tv_sec * SEC_TO_NSEC(1))); + clock_nanosleep(CLOCK_MONOTONIC, 0, &res, NULL); + } +} + +static inline void +platform_yield() +{ +} diff --git a/src/platform_linux/platform_sort.h b/src/platform_linux/platform_sort.h new file mode 100644 index 000000000..fb7d7b9f3 --- /dev/null +++ b/src/platform_linux/platform_sort.h @@ -0,0 +1,28 @@ +// Copyright 2018-2026 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include +#include + +/* + * The comparator follows the same conventions as that of qsort(3). Ie. if: + * a>b: return 1 + * a + +// Spin lock +typedef pthread_spinlock_t platform_spinlock; + +static inline platform_status +platform_spinlock_init(platform_spinlock *lock) +{ + int ret; + + // Init spinlock so it can be shared between processes, if so configured + ret = pthread_spin_init(lock, PTHREAD_PROCESS_SHARED); + return CONST_STATUS(ret); +} + + +static inline platform_status +platform_spinlock_destroy(platform_spinlock *lock) +{ + int ret; + + ret = pthread_spin_destroy(lock); + + return CONST_STATUS(ret); +} + +static inline platform_status +platform_spin_lock(platform_spinlock *lock) +{ + int ret; + + ret = pthread_spin_lock(lock); + + return CONST_STATUS(ret); +} + +static inline platform_status +platform_spin_unlock(platform_spinlock *lock) +{ + int ret; + + ret = pthread_spin_unlock(lock); + + return CONST_STATUS(ret); +} diff --git a/src/platform_linux/platform_status.h b/src/platform_linux/platform_status.h new file mode 100644 index 000000000..01ac55001 --- /dev/null +++ b/src/platform_linux/platform_status.h @@ -0,0 +1,57 @@ +// Copyright 2018-2026 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "splinterdb/platform_linux/public_platform.h" +#include +#include + +typedef typeof(EINVAL) internal_platform_status; + +// Platform status +typedef struct { + internal_platform_status r; +} platform_status; + +#define CONST_STATUS(status) ((const platform_status){.r = status}) + +// platform status +#define STATUS_OK CONST_STATUS(0) +#define STATUS_NO_MEMORY CONST_STATUS(ENOMEM) +#define STATUS_BUSY CONST_STATUS(EAGAIN) +#define STATUS_LIMIT_EXCEEDED CONST_STATUS(ENOSPC) +#define STATUS_NO_SPACE CONST_STATUS(ENOSPC) +#define STATUS_TIMEDOUT CONST_STATUS(ETIMEDOUT) +#define STATUS_NO_PERMISSION CONST_STATUS(EPERM) +#define STATUS_BAD_PARAM CONST_STATUS(EINVAL) +#define STATUS_INVALID_STATE CONST_STATUS(EINVAL) +#define STATUS_NOT_FOUND CONST_STATUS(ENOENT) +#define STATUS_IO_ERROR CONST_STATUS(EIO) +#define STATUS_NOTSUP CONST_STATUS(ENOTSUP) +#define STATUS_TEST_FAILED CONST_STATUS(-1) + +// platform predicates +static inline bool32 +STATUS_IS_EQ(const platform_status s1, const platform_status s2) +{ + return s1.r == s2.r; +} + +static inline bool32 +STATUS_IS_NE(const platform_status s1, const platform_status s2) +{ + return s1.r != s2.r; +} + +static inline bool32 +SUCCESS(const platform_status s) +{ + return STATUS_IS_EQ(s, STATUS_OK); +} + +static inline const char * +platform_status_to_string(const platform_status status) +{ + return strerror(status.r); +} diff --git a/src/platform_linux/platform_string.h b/src/platform_linux/platform_string.h new file mode 100644 index 000000000..fbf90c61e --- /dev/null +++ b/src/platform_linux/platform_string.h @@ -0,0 +1,21 @@ +// Copyright 2018-2026 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include +#include + +#define MAX_STRING_LENGTH 256 + +typedef struct { + char *token_str; + char *last_token; + int last_token_len; +} platform_strtok_ctx; + +static inline char * +platform_strtok_r(char *str, const char *delim, platform_strtok_ctx *ctx) +{ + return strtok_r(str, delim, &ctx->token_str); +} diff --git a/src/platform_linux/platform_threads.c b/src/platform_linux/platform_threads.c new file mode 100644 index 000000000..9f076cd18 --- /dev/null +++ b/src/platform_linux/platform_threads.c @@ -0,0 +1,433 @@ +// Copyright 2018-2026 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +#include "platform_threads.h" +#include "splinterdb/platform_linux/public_platform.h" +#include "platform_log.h" +#include +#include + +__thread threadid xxxtid = INVALID_TID; + +// xxxpid is valid iff ospid == getpid() +threadid xxxpid; +pid_t ospid; + + +/**************************************** + * Thread ID allocation and management * + ****************************************/ + +/* + * bitmask used for allocating thread id's. + * If a bit is set to 0, it means we have an in use thread id for that + * particular position, 1 means it is unset and that thread id is available + * for use. + */ +typedef struct threadid_allocator { + volatile uint64 bitmask_lock; + uint64 available_tids[(MAX_THREADS + 63) / 64]; + // number of threads allocated so far. + threadid num_threads; +} threadid_allocator; + +typedef struct pid_allocator { + volatile uint64 lock; + uint64 thread_count[MAX_THREADS]; +} pid_allocator; + +typedef struct thread_invocation { + platform_thread_worker worker; + void *arg; +} thread_invocation; + +typedef struct id_allocator { + threadid_allocator tid_allocator; + pid_allocator pid_allocator; + volatile uint64 process_event_callback_list_lock; + process_event_callback_list_node *process_event_callback_list; + thread_invocation thread_invocations[MAX_THREADS]; +} id_allocator; + +// This will be allocated in shared memory so that it is shared by all +// processes. +static id_allocator *id_alloc = NULL; + +/* + * task_init_tid_bitmask() - Initialize the global bitmask of active threads + * in the task system structure to indicate that no threads are currently + * active. + */ +static void +id_allocator_init_if_needed(void) +{ + if (id_alloc == NULL) { + id_allocator *my_id_alloc = NULL; + my_id_alloc = mmap(NULL, + sizeof(id_allocator), + PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_ANONYMOUS, + -1, + 0); + if (my_id_alloc == MAP_FAILED) { + platform_error_log("Failed to allocate memory for id allocator"); + return; + } + memset(my_id_alloc, 0x00, sizeof(id_allocator)); + memset(my_id_alloc->tid_allocator.available_tids, + 0xFF, + sizeof(my_id_alloc->tid_allocator.available_tids)); + if (!__sync_bool_compare_and_swap(&id_alloc, NULL, my_id_alloc)) { + munmap(my_id_alloc, sizeof(id_allocator)); + } + } +} + +/* + * Allocate a threadid. Returns INVALID_TID when no tid is available. + */ +static threadid +allocate_threadid() +{ + threadid tid = INVALID_TID; + uint64 *tid_bitmask = id_alloc->tid_allocator.available_tids; + uint64 old_bitmask; + uint64 new_bitmask; + + while (__sync_lock_test_and_set(&id_alloc->tid_allocator.bitmask_lock, 1)) { + // spin + } + + int i; + uint64 pos = 0; + for (i = 0; pos == 0 && i < (MAX_THREADS + 63) / 64; i++) { + old_bitmask = tid_bitmask[i]; + // first bit set to 1 starting from LSB. + pos = __builtin_ffsl(old_bitmask); + } + + if (pos == 0) { + __sync_lock_release(&id_alloc->tid_allocator.bitmask_lock); + platform_error_log("No thread id available"); + return INVALID_TID; + } + + i--; + + // builtin_ffsl returns the position plus 1. + tid = pos - 1; + // set bit at that position to 0, indicating in use. + new_bitmask = (old_bitmask & ~(1ULL << tid)); + int r = __sync_bool_compare_and_swap( + &id_alloc->tid_allocator.available_tids[i], old_bitmask, new_bitmask); + platform_assert(r); + + __sync_lock_release(&id_alloc->tid_allocator.bitmask_lock); + + tid += i * 64; + + // Invariant: we have successfully allocated tid + + // atomically increment the num_threads. + uint64 tmp = __sync_fetch_and_add(&id_alloc->tid_allocator.num_threads, 1); + platform_assert(tmp < MAX_THREADS); + + return tid; +} + +/* + * De-registering a task frees up the thread's index so that it can be + * re-used. + */ +static void +deallocate_threadid(threadid tid) +{ + uint64 *tid_bitmask = id_alloc->tid_allocator.available_tids; + + while (__sync_lock_test_and_set(&id_alloc->tid_allocator.bitmask_lock, 1)) { + // spin + } + + uint64 bitmask_val = tid_bitmask[tid / 64]; + + // Ensure that caller is only clearing for a thread that's in-use. + platform_assert(!(bitmask_val & (1ULL << (tid % 64))), + "Thread [%lu] is expected to be in-use. Bitmap: 0x%lx", + tid, + bitmask_val); + + // set bit back to 1 to indicate a free slot. + uint64 tmp_bitmask = tid_bitmask[tid / 64]; + uint64 new_value = tmp_bitmask | (1ULL << (tid % 64)); + while (!__sync_bool_compare_and_swap( + tid_bitmask + tid / 64, tmp_bitmask, new_value)) + { + tmp_bitmask = tid_bitmask[tid / 64]; + new_value = tmp_bitmask | (1ULL << (tid % 64)); + } + + __sync_lock_release(&id_alloc->tid_allocator.bitmask_lock); + + // atomically decrement the num_threads. + uint64 tmp = __sync_fetch_and_sub(&id_alloc->tid_allocator.num_threads, 1); + platform_assert(0 < tmp); + + xxxtid = INVALID_TID; +} + +static platform_status +ensure_xxxpid_is_setup(void) +{ + pid_t myospid = getpid(); + + while (__sync_lock_test_and_set(&id_alloc->pid_allocator.lock, 1)) { + // spin + } + + if (myospid == ospid) { + platform_assert(xxxpid < INVALID_TID); + id_alloc->pid_allocator.thread_count[xxxpid]++; + __sync_lock_release(&id_alloc->pid_allocator.lock); + return STATUS_OK; + } + + for (int i = 0; i < MAX_THREADS; i++) { + if (id_alloc->pid_allocator.thread_count[i] == 0) { + id_alloc->pid_allocator.thread_count[i] = 1; + xxxpid = i; + ospid = myospid; + xxxtid = INVALID_TID; + + break; + } + } + + __sync_lock_release(&id_alloc->pid_allocator.lock); + + if (ospid != myospid) { + return STATUS_BUSY; + } + + return STATUS_OK; +} + +static void +decref_xxxpid(void) +{ + while (__sync_lock_test_and_set(&id_alloc->pid_allocator.lock, 1)) { + // spin + } + + platform_assert(xxxpid < INVALID_TID); + + id_alloc->pid_allocator.thread_count[xxxpid]--; + if (id_alloc->pid_allocator.thread_count[xxxpid] == 0) { + + while (__sync_lock_test_and_set( + &id_alloc->process_event_callback_list_lock, 1)) + { + // spin + } + process_event_callback_list_node *current = + id_alloc->process_event_callback_list; + while (current != NULL) { + current->termination(xxxpid, current->arg); + current = current->next; + } + __sync_lock_release(&id_alloc->process_event_callback_list_lock); + + xxxpid = INVALID_TID; + ospid = 0; + } + + __sync_lock_release(&id_alloc->pid_allocator.lock); +} + +void +platform_linux_add_process_event_callback( + process_event_callback_list_node *node) +{ + id_allocator_init_if_needed(); + while ( + __sync_lock_test_and_set(&id_alloc->process_event_callback_list_lock, 1)) + { + // spin + } + node->next = id_alloc->process_event_callback_list; + id_alloc->process_event_callback_list = node; + __sync_lock_release(&id_alloc->process_event_callback_list_lock); +} + +void +platform_linux_remove_process_event_callback( + process_event_callback_list_node *node) +{ + while ( + __sync_lock_test_and_set(&id_alloc->process_event_callback_list_lock, 1)) + { + // spin + } + + process_event_callback_list_node *current = + id_alloc->process_event_callback_list; + process_event_callback_list_node *prev = NULL; + while (current != NULL) { + if (current == node) { + if (prev == NULL) { + id_alloc->process_event_callback_list = current->next; + } else { + prev->next = current->next; + } + break; + } + prev = current; + current = current->next; + } + + __sync_lock_release(&id_alloc->process_event_callback_list_lock); +} + +/****************************************************************************** + * Registering/deregistering threads. + ******************************************************************************/ + +/* + * platform_deregister_thread() - Deregister an active thread. + * + * Deregistration involves clearing the thread ID (index) for this thread. + */ +void +platform_deregister_thread() +{ + threadid tid = platform_get_tid(); + + platform_assert(tid != INVALID_TID, + "Error! Attempt to deregister unregistered thread.\n"); + + deallocate_threadid(tid); + decref_xxxpid(); +} + +static void +thread_registration_cleanup_function(void *arg) +{ + if (xxxtid == INVALID_TID) { + platform_error_log("Thread registration cleanup function called for " + "unregistered thread %lu", + xxxtid); + } else { + platform_deregister_thread(); + } +} + + +/* + * platform_register_thread(): Register this new thread. + * + * Registration implies: + * - Acquire a new thread ID (index) for this to-be-active thread + */ +int +platform_register_thread(void) +{ + id_allocator_init_if_needed(); + + platform_status status = ensure_xxxpid_is_setup(); + if (!SUCCESS(status)) { + return -1; + } + + threadid thread_tid = xxxtid; + + // Before registration, all SplinterDB threads' tid will be its default + // value; i.e. INVALID_TID. + platform_assert(thread_tid == INVALID_TID, + "Attempt to register thread that is already " + "registered as thread %lu\n", + thread_tid); + + thread_tid = allocate_threadid(); + // Unavailable threads is a temporary state that could go away. + if (thread_tid == INVALID_TID) { + decref_xxxpid(); + return -1; + } + + platform_assert(thread_tid < MAX_THREADS); + xxxtid = thread_tid; + + return 0; +} + + +/****************************************************************************** + * Thread creation and joining. + ******************************************************************************/ + +static void * +thread_worker_function(void *arg) +{ + pthread_cleanup_push(thread_registration_cleanup_function, NULL); + thread_invocation *thread_inv = (thread_invocation *)arg; + threadid tid = thread_inv - id_alloc->thread_invocations; + xxxtid = tid; + thread_inv->worker(thread_inv->arg); + pthread_cleanup_pop(1); + return NULL; +} + +/* + * platform_thread_create() - External interface to create a Splinter thread. + */ +platform_status +platform_thread_create(platform_thread *thread, + bool32 detached, + platform_thread_worker worker, + void *arg, + platform_heap_id heap_id) +{ + int ret; + + id_allocator_init_if_needed(); + platform_status rc = ensure_xxxpid_is_setup(); + if (!SUCCESS(rc)) { + return rc; + } + + // We allocate the threadid here, rather than in the thread_worker_function, + // so that we can report an error if the threadid allocation fails. + threadid tid = allocate_threadid(); + if (tid == INVALID_TID) { + return STATUS_BUSY; + } + thread_invocation *thread_inv = &id_alloc->thread_invocations[tid]; + thread_inv->worker = worker; + thread_inv->arg = arg; + + ret = pthread_create(thread, NULL, thread_worker_function, thread_inv); + if (ret != 0) { + deallocate_threadid(tid); + decref_xxxpid(); + return STATUS_NO_MEMORY; + } + + return CONST_STATUS(ret); +} + +platform_status +platform_thread_join(platform_thread *thread) +{ + int ret; + void *retval; + + ret = pthread_join(*thread, &retval); + + return CONST_STATUS(ret); +} + +threadid +platform_num_threads(void) +{ + id_allocator_init_if_needed(); + return id_alloc->tid_allocator.num_threads; +} diff --git a/src/platform_linux/platform_threads.h b/src/platform_linux/platform_threads.h new file mode 100644 index 000000000..9eb06d7f5 --- /dev/null +++ b/src/platform_linux/platform_threads.h @@ -0,0 +1,76 @@ +// Copyright 2018-2026 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "splinterdb/platform_linux/public_platform.h" +#include "platform_status.h" +#include "platform_heap.h" +#include +#include + +/* + * MAX_THREADS is used primarily for convenience, where allocations made on a + * per-thread basis create an array with MAX_THREADS items, e.g. the + * trunk_stats field in trunk_handle. The task subsystem also uses a 64-bit + * bit-array to track thread IDs in use. This could be changed relatively + * easily if needed. + */ +#define MAX_THREADS (64) +#define INVALID_TID (MAX_THREADS) + + +static inline threadid +platform_get_tid() +{ + extern __thread threadid xxxtid; + return xxxtid; +} + +/* This is not part of the platform API. It is used internally to this platform + * implementation. Specifically, it is used in laio.c. */ +static inline threadid +platform_linux_get_pid() +{ + extern threadid xxxpid; + return xxxpid; +} + +typedef void (*process_event_callback)(threadid, void *); + +typedef struct process_event_callback_list_node { + process_event_callback termination; + void *arg; + struct process_event_callback_list_node *next; +} process_event_callback_list_node; + +void +platform_linux_add_process_event_callback( + process_event_callback_list_node *node); + +void +platform_linux_remove_process_event_callback( + process_event_callback_list_node *node); + +static inline int +platform_get_os_pid() +{ + return getpid(); +} + +typedef void (*platform_thread_worker)(void *); + +typedef pthread_t platform_thread; + +platform_status +platform_thread_create(platform_thread *thread, + bool32 detached, + platform_thread_worker worker, + void *arg, + platform_heap_id heap_id); + +platform_status +platform_thread_join(platform_thread *thread); + +threadid +platform_num_threads(void); diff --git a/src/platform_linux/platform_time.h b/src/platform_linux/platform_time.h new file mode 100644 index 000000000..b3223a6dc --- /dev/null +++ b/src/platform_linux/platform_time.h @@ -0,0 +1,40 @@ +// Copyright 2018-2026 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "splinterdb/platform_linux/public_platform.h" +#include "platform_units.h" +#include + +#pragma once + +static inline timestamp +platform_get_timestamp(void) +{ + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return SEC_TO_NSEC(ts.tv_sec) + ts.tv_nsec; +} + +static inline timestamp +platform_timestamp_elapsed(timestamp tv) +{ + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return SEC_TO_NSEC(ts.tv_sec) + ts.tv_nsec - tv; +} + +static inline timestamp +platform_timestamp_diff(timestamp start, timestamp end) +{ + return end - start; +} + +static inline timestamp +platform_get_real_time(void) +{ + struct timespec ts; + clock_gettime(CLOCK_REALTIME, &ts); + return SEC_TO_NSEC(ts.tv_sec) + ts.tv_nsec; +} diff --git a/src/platform_linux/platform_typed_alloc.h b/src/platform_linux/platform_typed_alloc.h new file mode 100644 index 000000000..d7a0cfb21 --- /dev/null +++ b/src/platform_linux/platform_typed_alloc.h @@ -0,0 +1,227 @@ +// Copyright 2018-2026 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "splinterdb/platform_linux/public_platform.h" +#include "platform_assert.h" +#include "platform_util.h" +#include "platform_heap.h" + +/* + * ----------------------------------------------------------------------------- + * TYPED_MANUAL_MALLOC(), TYPED_MANUAL_ZALLOC() - + * TYPED_ARRAY_MALLOC(), TYPED_ARRAY_ZALLOC() - + * + * Utility macros to avoid common memory allocation / initialization mistakes. + * NOTE: ZALLOC variants will also memset allocated memory chunk to 0. + * + * Call-flow is: + * TYPED_MALLOC() + * -> TYPED_ARRAY_MALLOC() + * -> TYPED_MANUAL_MALLOC() -> platform_aligned_malloc() + * + * TYPED_ZALLOC() + * -> TYPED_ARRAY_ZALLOC() + * -> TYPED_MANUAL_ZALLOC() -> platform_aligned_zalloc() + * + * ----------------------------------------------------------------------------- + * Common mistake to make is: + * TYPE *foo = platform_malloc(sizeof(WRONG_TYPE)); + * or + * TYPE *foo; + * ... + * foo = platform_malloc(sizeof(WRONG_TYPE)); + * WRONG_TYPE may have been the correct type and the code changed, or you may + * have the wrong number of '*'s, or you may have copied and pasted and forgot + * to change the type/size. + * + * A useful pattern is: + * TYPE *foo = platform_malloc(sizeof(*foo)) + * but you can still cause a mistake by not typing the variable name correctly + * twice. + * + * We _could_ make a macro MALLOC_AND_SET for the above pattern, e.g. + * define MALLOC_AND_SET(x) (x) = platform_malloc(sizeof(*x)) + * but that is hard to read. + * + * The boilerplate of extra typing (e.g. remember to type x twice) isn't a big + * problem, it's just that you can get the types wrong and the compiler won't + * notice. + * We can keep the easy-to-read pattern of `x = function(x)` with typesafety by + * including a cast inside the macro. + * + * These macros let you avoid the avoid the mistakes by typing malloc. + * + * struct foo *foo_pointer = TYPED_MALLOC(foo_pointer); + * struct foo *foo_pointer2; + * foo_pointer2 = TYPED_MALLOC(foo_pointer2); + * + * To replace dynamic/vla arrays, use TYPED_ARRAY_MALLOC, e.g. + * struct foo array[X()]; + * becomes + * struct foo *array = TYPED_ARRAY_MALLOC(array, X()); + * ZALLOC versions will also memset to 0. + * + * All mallocs here are cache aligned. + * Consider the following types: + * struct unaligned_foo { + * cache_aligned_type bar; + * }; + * If you (unaligned) malloc a 'unaligned_foo', the compile is still allowed to + * assume that bar is properly cache aligned. It may do unsafe optimizations. + * One known unsafe optimization is turning memset(...,0,...) into avx + * instructions that crash if something is not aligned. + * + * The simplest solution is to simply align ALL mallocs. + * We do not do a sufficient number of mallocs for this to have any major + * problems. The slight additional memory usage (for tiny mallocs) should not + * matter. The potential minor perf hit should also not matter due to us + * slowly coalescing all mallocs anyway into either initialization or amortized + * situations. + * + * Alternative solutions are to be careful with mallocs, and/or make ALL structs + * be aligned. + * + * Another common use case is if you have a struct with a flexible array member. + * In that case you should use TYPED_FLEXIBLE_STRUCT_(M|Z)ALLOC + * + * If you are doing memory size calculation manually (e.g. if you're avoiding + * multiple mallocs by doing one larger malloc and setting pointers manually, + * or the data type has a something[] or something[0] at the end) you should + * instead use the TYPED_*ALLOC_MANUAL macros that allow you to provide the + * exact size. These macros currently assume (and in debug mode assert) that + * you will never malloc LESS than the struct/type size. + * + * DO NOT USE these macros to assign to a void*. The debug asserts will cause + * a compile error when debug is on. Assigning to a void* should be done by + * calling aligned_alloc manually (or create a separate macro) + * + * Parameters: + * hid - Platform heap-ID to allocate memory from. + * v - Structure to allocate memory for. + * n - Number of bytes of memory to allocate. + * ----------------------------------------------------------------------------- + */ +#define TYPED_MANUAL_MALLOC(hid, v, n) \ + ({ \ + debug_assert((n) >= sizeof(*(v))); \ + (typeof(v))platform_aligned_malloc(hid, \ + PLATFORM_CACHELINE_SIZE, \ + (n), \ + STRINGIFY(v), \ + __func__, \ + __FILE__, \ + __LINE__); \ + }) +#define TYPED_MANUAL_ZALLOC(hid, v, n) \ + ({ \ + debug_assert((n) >= sizeof(*(v))); \ + (typeof(v))platform_aligned_zalloc(hid, \ + PLATFORM_CACHELINE_SIZE, \ + (n), \ + STRINGIFY(v), \ + __func__, \ + __FILE__, \ + __LINE__); \ + }) + +/* + * TYPED_ALIGNED_MALLOC(), TYPED_ALIGNED_ZALLOC() + * + * Allocate memory for a typed structure at caller-specified alignment. + * These are similar to TYPED_MANUAL_MALLOC() & TYPED_MANUAL_ZALLOC() but with + * the difference that the alignment is caller-specified. + * + * Parameters: + * hid - Platform heap-ID to allocate memory from. + * a - Alignment needed for allocated memory. + * v - Structure to allocate memory for. + * n - Number of bytes of memory to allocate. + */ +#define TYPED_ALIGNED_MALLOC(hid, a, v, n) \ + ({ \ + debug_assert((n) >= sizeof(*(v))); \ + (typeof(v))platform_aligned_malloc( \ + hid, (a), (n), STRINGIFY(v), __func__, __FILE__, __LINE__); \ + }) +#define TYPED_ALIGNED_ZALLOC(hid, a, v, n) \ + ({ \ + debug_assert((n) >= sizeof(*(v))); \ + (typeof(v))platform_aligned_zalloc( \ + hid, (a), (n), STRINGIFY(v), __func__, __FILE__, __LINE__); \ + }) + +/* + * FLEXIBLE_STRUCT_SIZE(): Compute the size of a structure 'v' with a nested + * flexible array member, array_field_name, with 'n' members. + * + * Flexible array members don't necessarily start after sizeof(v) + * They can start within the padding at the end, so the correct size + * needed to allocate a struct with a flexible array member is the + * larger of sizeof(struct v) or (offset of flexible array + + * n*sizeof(arraymember)) + * + * The only reasonable static assert we can do is check that the flexible array + * member is actually an array. We cannot check size==0 (compile error), and + * since it doesn't necessarily start at the end we also cannot check + * offset==sizeof. + * + * Parameters: + * v - Structure to allocate memory for. + * array_field_name - Name of flexible array field nested in 'v' + * n - Number of members in array_field_name[]. + */ +#define FLEXIBLE_STRUCT_SIZE(v, array_field_name, n) \ + ({ \ + _Static_assert(IS_ARRAY((v)->array_field_name), \ + "flexible array members must be arrays"); \ + max_size_t(sizeof(*(v)), \ + (n) * sizeof((v)->array_field_name[0]) \ + + offsetof(typeof(*(v)), array_field_name)); \ + }) + +/* + * ----------------------------------------------------------------------------- + * TYPED_FLEXIBLE_STRUCT_MALLOC(), TYPED_FLEXIBLE_STRUCT_ZALLOC() - + * Allocate memory for a structure with a nested flexible array member. + * + * Parameters: + * hid - Platform heap-ID to allocate memory from. + * v - Structure to allocate memory for. + * array_field_name - Name of flexible array field nested in 'v' + * n - Number of members in array_field_name[]. + * ----------------------------------------------------------------------------- + */ +#define TYPED_FLEXIBLE_STRUCT_MALLOC(hid, v, array_field_name, n) \ + TYPED_MANUAL_MALLOC( \ + hid, (v), FLEXIBLE_STRUCT_SIZE((v), array_field_name, (n))) + +#define TYPED_FLEXIBLE_STRUCT_ZALLOC(hid, v, array_field_name, n) \ + TYPED_MANUAL_ZALLOC( \ + hid, (v), FLEXIBLE_STRUCT_SIZE((v), array_field_name, (n))) + +/* + * TYPED_ARRAY_MALLOC(), TYPED_ARRAY_ZALLOC() + * Allocate memory for an array of 'n' elements of structure 'v'. + * + * Parameters: + * hid - Platform heap-ID to allocate memory from. + * v - Structure to allocate memory for. + * n - Number of members of type 'v' in array. + */ +#define TYPED_ARRAY_MALLOC(hid, v, n) \ + TYPED_MANUAL_MALLOC(hid, (v), (n) * sizeof(*(v))) +#define TYPED_ARRAY_ZALLOC(hid, v, n) \ + TYPED_MANUAL_ZALLOC(hid, (v), (n) * sizeof(*(v))) + +/* + * TYPED_ARRAY_MALLOC(), TYPED_ARRAY_ZALLOC() + * Allocate memory for one element of structure 'v'. + * + * Parameters: + * hid - Platform heap-ID to allocate memory from. + * v - Structure to allocate memory for. + */ +#define TYPED_MALLOC(hid, v) TYPED_ARRAY_MALLOC(hid, v, 1) +#define TYPED_ZALLOC(hid, v) TYPED_ARRAY_ZALLOC(hid, v, 1) diff --git a/src/platform_linux/platform_types.h b/src/platform_linux/platform_types.h deleted file mode 100644 index 7fc63d315..000000000 --- a/src/platform_linux/platform_types.h +++ /dev/null @@ -1,187 +0,0 @@ -// Copyright 2018-2021 VMware, Inc. -// SPDX-License-Identifier: Apache-2.0 - -#ifndef PLATFORM_LINUX_TYPES_H -#define PLATFORM_LINUX_TYPES_H - -#include -#include // for isspace,isascii,isdigit,isalpha,isupper -#include -#include -#include -#include -#include -#include -#include -#include -#include - -// platform status -typedef typeof(EINVAL) internal_platform_status; - -#define STATUS_OK CONST_STATUS(0) -#define STATUS_NO_MEMORY CONST_STATUS(ENOMEM) -#define STATUS_BUSY CONST_STATUS(EAGAIN) -#define STATUS_LIMIT_EXCEEDED CONST_STATUS(ENOSPC) -#define STATUS_NO_SPACE CONST_STATUS(ENOSPC) -#define STATUS_TIMEDOUT CONST_STATUS(ETIMEDOUT) -#define STATUS_NO_PERMISSION CONST_STATUS(EPERM) -#define STATUS_BAD_PARAM CONST_STATUS(EINVAL) -#define STATUS_INVALID_STATE CONST_STATUS(EINVAL) -#define STATUS_NOT_FOUND CONST_STATUS(ENOENT) -#define STATUS_IO_ERROR CONST_STATUS(EIO) -#define STATUS_NOTSUP CONST_STATUS(ENOTSUP) -#define STATUS_TEST_FAILED CONST_STATUS(-1) - -// checksums -typedef XXH32_hash_t checksum32; -typedef XXH64_hash_t checksum64; -typedef XXH128_hash_t checksum128; - -#define PLATFORM_CACHELINE_SIZE 64 -#define PLATFORM_CACHELINE_ALIGNED \ - __attribute__((__aligned__(PLATFORM_CACHELINE_SIZE))) - -/* - * Helper macro that causes branch prediction to favour the likely - * side of a jump instruction. If the prediction is correct, - * the jump instruction takes zero cycles. If it's wrong, the - * processor pipeline needs to be flushed and it can cost - * several cycles. - */ -#define LIKELY(_exp) __builtin_expect(!!(_exp), 1) -#define UNLIKELY(_exp) __builtin_expect(!!(_exp), 0) - -/* - * A handle which buffers streamed content to be atomically written to a - * platform_log_handle. - */ -typedef struct { - char *str; - size_t size; - FILE *stream; -} platform_stream_handle; - -typedef sem_t platform_semaphore; - -#define STRINGIFY(x) #x -#define STRINGIFY_VALUE(s) STRINGIFY(s) -#define FRACTION_FMT(w, s) "%" STRINGIFY_VALUE(w) "." STRINGIFY_VALUE(s) "f" -#define FRACTION_ARGS(f) ((double)(f).numerator / (double)(f).denominator) - -typedef pthread_t platform_thread; - -// Thread-specific mutex, with ownership tracking. -typedef struct { - pthread_mutex_t mutex; - threadid owner; -} platform_mutex; - -// Spin lock -typedef pthread_spinlock_t platform_spinlock; - -// Distributed Batch RW Lock -typedef struct { - volatile uint8 lock; - volatile uint8 claim; -} platform_claimlock; - -typedef struct { - platform_claimlock write_lock[PLATFORM_CACHELINE_SIZE / 2]; - volatile uint8 read_counter[MAX_THREADS][PLATFORM_CACHELINE_SIZE / 2]; -} PLATFORM_CACHELINE_ALIGNED platform_batch_rwlock; - -_Static_assert(sizeof(platform_batch_rwlock) - == PLATFORM_CACHELINE_SIZE * (MAX_THREADS / 2 + 1), - "Missized platform_batch_rwlock\n"); - - -/* - * The state machine for a thread interacting with a batch_rwlock is: - * - * get claim lock - * unlocked <-------> read-locked <----------> claimed <--------> write-locked - * unget unclaim unlock - * - * Note that try_claim() may fail, in which case the state of the lock - * is unchanged, i.e. the caller still holds a read lock. - */ - - -void -platform_batch_rwlock_init(platform_batch_rwlock *lock); - -void -platform_batch_rwlock_deinit(platform_batch_rwlock *lock); - -/* no lock -> shared lock */ -void -platform_batch_rwlock_get(platform_batch_rwlock *lock, uint64 lock_idx); - -/* shared lock -> no lock */ -void -platform_batch_rwlock_unget(platform_batch_rwlock *lock, uint64 lock_idx); - -/* - * shared-lock -> claim (may fail) - * - * Callers still hold a shared lock after a failed claim attempt. - * Callers _must_ release their shared lock after a failed claim attempt. - */ -bool32 -platform_batch_rwlock_try_claim(platform_batch_rwlock *lock, uint64 lock_idx); - -/* shared-lock -> claim, BUT(!) may temporarily release the shared-lock in the - * process. */ -void -platform_batch_rwlock_claim_loop(platform_batch_rwlock *lock, uint64 lock_idx); - -/* claim -> shared lock */ -void -platform_batch_rwlock_unclaim(platform_batch_rwlock *lock, uint64 lock_idx); - -/* claim -> exclusive lock */ -void -platform_batch_rwlock_lock(platform_batch_rwlock *lock, uint64 lock_idx); - -/* exclusive lock -> claim */ -void -platform_batch_rwlock_unlock(platform_batch_rwlock *lock, uint64 lock_idx); - -/* exclusive-lock -> unlocked */ -void -platform_batch_rwlock_full_unlock(platform_batch_rwlock *lock, uint64 lock_idx); - - -// Buffer handle -typedef struct { - void *addr; - size_t length; -} buffer_handle; - -// iohandle for laio -typedef struct laio_handle platform_io_handle; - -typedef void *platform_module_id; -typedef void *platform_heap_id; - -typedef struct { - unsigned int num_buckets; - const long *bucket_limits; - long min, max, total; - unsigned long num; // no. of elements - unsigned long count[]; -} *platform_histo_handle; - -#define UNUSED_PARAM(_parm) _parm __attribute__((__unused__)) -#define UNUSED_TYPE(_parm) UNUSED_PARAM(_parm) - -#define ROUNDUP(x, y) (((x) + (y)-1) / (y) * (y)) -#define ROUNDDOWN(x, y) ((x) / (y) * (y)) - -typedef struct platform_condvar { - pthread_mutex_t lock; - pthread_cond_t cond; -} platform_condvar; - -#endif /* PLATFORM_LINUX_TYPES_H */ diff --git a/src/platform_linux/platform_units.h b/src/platform_linux/platform_units.h new file mode 100644 index 000000000..aef598af5 --- /dev/null +++ b/src/platform_linux/platform_units.h @@ -0,0 +1,44 @@ +// Copyright 2018-2026 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +/* Can probably be moved into a platform_common directory */ + +#pragma once + +// Data unit constants +#define KiB (1024UL) +#define MiB (KiB * 1024) +#define GiB (MiB * 1024) +#define TiB (GiB * 1024) + +// Convert 'x' in unit-specifiers to bytes +#define KiB_TO_B(x) ((x) * KiB) +#define MiB_TO_B(x) ((x) * MiB) +#define GiB_TO_B(x) ((x) * GiB) +#define TiB_TO_B(x) ((x) * TiB) + +// Convert 'x' in bytes to 'int'-value with unit-specifiers +#define B_TO_KiB(x) ((x) / KiB) +#define B_TO_MiB(x) ((x) / MiB) +#define B_TO_GiB(x) ((x) / GiB) +#define B_TO_TiB(x) ((x) / TiB) + +// For x bytes, returns as int the fractional portion modulo a unit-specifier +#define B_TO_KiB_FRACT(x) ((100 * ((x) % KiB)) / KiB) +#define B_TO_MiB_FRACT(x) ((100 * ((x) % MiB)) / MiB) +#define B_TO_GiB_FRACT(x) ((100 * ((x) % GiB)) / GiB) +#define B_TO_TiB_FRACT(x) ((100 * ((x) % TiB)) / TiB) + +// Time unit constants +#define THOUSAND (1000UL) +#define MILLION (THOUSAND * THOUSAND) +#define BILLION (THOUSAND * MILLION) + +#define USEC_TO_SEC(x) ((x) / MILLION) +#define USEC_TO_NSEC(x) ((x) * THOUSAND) +#define NSEC_TO_SEC(x) ((x) / BILLION) +#define NSEC_TO_MSEC(x) ((x) / MILLION) +#define NSEC_TO_USEC(x) ((x) / THOUSAND) +#define SEC_TO_MSEC(x) ((x) * THOUSAND) +#define SEC_TO_USEC(x) ((x) * MILLION) +#define SEC_TO_NSEC(x) ((x) * BILLION) diff --git a/src/platform_linux/platform_util.h b/src/platform_linux/platform_util.h new file mode 100644 index 000000000..67ea5a575 --- /dev/null +++ b/src/platform_linux/platform_util.h @@ -0,0 +1,266 @@ +// Copyright 2018-2026 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +/* + * This file could likely be moved into a platform_common directory that could + * serve as a common library for all platforms. + */ + +#pragma once +#include "splinterdb/platform_linux/public_platform.h" +#include + +#define IS_POWER_OF_2(n) ((n) > 0 && ((n) & ((n) - 1)) == 0) + +#ifndef MAX +# define MAX(a, b) ((a) > (b) ? (a) : (b)) +#endif + +#ifndef MIN +# define MIN(a, b) ((a) < (b) ? (a) : (b)) +#endif + +static inline size_t +max_size_t(size_t a, size_t b) +{ + return a > b ? a : b; +} + +/* calculate difference between two pointers */ +static inline ptrdiff_t +diff_ptr(const void *base, const void *limit) +{ + _Static_assert(sizeof(char) == 1, "Assumption violated"); + return (char *)limit - (char *)base; +} + +/* + * Helper macro that takes a pointer, type of the container, and the + * name of the member the pointer refers to. The macro expands to a + * new address pointing to the container which accomodates the + * specified member. + */ +#ifndef container_of +# define container_of(ptr, type, memb) \ + ((type *)((char *)(ptr) - offsetof(type, memb))) +#endif + +/* + * C11 and higher already supports native _Static_assert which has good + * compiler output on errors + * Since we are lower than C11, we need something that works. + */ +#if !defined(__STDC_VERSION__) || __STDC_VERSION__ < 201112l +# define _Static_assert(expr, str) \ + do { \ + typedef char oc_assert_fail[((expr) ? 1 : -1)] \ + __attribute__((__unused__)); \ + } while (0) +#endif + + +// Helper function so that ARRAYSIZE can be used inside of _Static_assert +#define ASSERT_EXPR(condition, return_value) \ + (sizeof(char[(condition) ? 1 : -1]) ? (return_value) : (return_value)) + +/* + * Utility macro to test if an indexable expression is an array. + * Gives an error at compile time if the expression is not indexable. + * Only pointers and arrays are indexable. + * Expression value is 0 for pointer and 1 for array + */ +#define IS_ARRAY(x) \ + __builtin_choose_expr( \ + __builtin_types_compatible_p(typeof((x)[0])[], typeof(x)), 1, 0) + +/* + * Errors at compile time if you use ARRAY_SIZE() on a pointer. + * ARRAY_SIZE can be used inside of _Static_assert. + */ +#define ARRAY_SIZE(x) ASSERT_EXPR(IS_ARRAY(x), (sizeof(x) / sizeof((x)[0]))) + +/* + * ----------------------------------------------------------------------------- + * Utility macros to clear memory + * They have similar usage/prevent similar mistakes to the TYPED_MALLOC + * kind of macros. + * They are not appropriate when you have a malloced array or when you've + * done non-obvious sized mallocs. + * + * Array/Pointer/Struct have different implementations cause the size + * and pointer calculations are different. + * Passing any one type to a clearing function of another type is likely + * to be a bug, so if the calling isn't perfect it will give a compile-time + * error. + * ----------------------------------------------------------------------------- + */ +/* + * Zero an array. + * Cause compile-time error if used on pointer or non-indexible variable + */ +#define ZERO_ARRAY(v) \ + do { \ + _Static_assert(IS_ARRAY(v), "Use of ZERO_ARRAY on non-array object"); \ + memset((v), 0, sizeof(v)); \ + } while (0) + +/* + * Zero a manual array (e.g. we malloced an array). + * Cause compile-time error if used on an array or non-indexible variable + */ +#define ZERO_CONTENTS_N(v, n) \ + do { \ + _Static_assert(!IS_ARRAY(v), "Use of ZERO_CONTENTS on array"); \ + memset((v), 0, (n) * sizeof(*(v))); \ + } while (0) + +/* + * Zero a non-array pointer (clears what the pointer points to). + * Cause compile-time error if used on an array or non-indexible variable + * + * Should not be used to zero out structs. Use ZERO_STRUCT instead. + * It is difficult to add compile errors when you pass structs here, but + * a debug compile is likely to compile error on the debug_assert. + */ +#define ZERO_CONTENTS(v) ZERO_CONTENTS_N((v), 1) + +/* + * Zero a struct. + * We want to give a compile-time error if v is not a struct, so we cannot + * use something like memset(&v, 0, sizeof(v)); Even C11 is not rich enough + * to determine if a variable is a struct, so we use syntax errors to catch + * it instead. + * + * Unfortunately C11/gnu11 syntax is only rich enough to cause errors on + * pointers and primitives, but not arrays. + * + * Note: We could take advantage of the syntax and still use memset like this: + * __builtin_choose_expr( + * 0, + * (typeof(v)) {}, + * memset(&(v), 0, sizeof(v))) + * however while the above still does prevent pointers and primitives, it will + * incorrectly initialize arrays (read: corrupt memory) if they happen to get + * passed in. + * Struct assignment properly initializes both arrays and structs: + * v = (typeof(v)) {} + * Memset for structs: + * memset(&v, 0, sizeof(v)) + * Memset for arrays: + * memset(v, 0, sizeof(v)) + * + * memset performance is affected by compiler and headers + * struct assignment performance is just affected by compiler + * It's not obvious that one has a performance advantage and since memset is + * often a compiler intrinsic it's likely to have the same performance. + * + * Note; This version would only work during declaration: + * (v) = {} + * Note; This version could work during declaration and a regular statement, + * but we force it as a statement to match the usage of ZERO_ARRAY/ZERO_POINTER + * (v) = (typeof(v)) {} + * + * This macro intentionally CANNOT be used during declaration + * (see ZERO_STRUCT_AT_DECL). + */ +#define ZERO_STRUCT(v) \ + do { \ + (v) = (typeof(v)){}; \ + } while (0) + +/* + * Zero a struct at declaration time. + * Equivalent to doing: + * struct foo s; + * ZERO_STRUCT(s); + * Usage example: + * struct foo ZERO_STRUCT_AT_DECL(s); + * See documentation for ZERO_STRUCT. + * Note: + * You can actually use ZERO_STRUCT_AT_DECL as a regular statement, + * but it is slightly less safe because the first v cannot be wrapped + * in parenthesis. + */ +#define ZERO_STRUCT_AT_DECL(v) \ + v = (typeof(v)) {} + +#define UNUSED_PARAM(_parm) _parm __attribute__((__unused__)) +#define UNUSED_TYPE(_parm) UNUSED_PARAM(_parm) + +/* + * Helper macro that causes branch prediction to favour the likely + * side of a jump instruction. If the prediction is correct, + * the jump instruction takes zero cycles. If it's wrong, the + * processor pipeline needs to be flushed and it can cost + * several cycles. + */ +#define LIKELY(_exp) __builtin_expect(!!(_exp), 1) +#define UNLIKELY(_exp) __builtin_expect(!!(_exp), 0) + +#define STRINGIFY(x) #x +#define STRINGIFY_VALUE(s) STRINGIFY(s) + +#define ROUNDUP(x, y) (((x) + (y) - 1) / (y) * (y)) +#define ROUNDDOWN(x, y) ((x) / (y) * (y)) + +typedef struct fraction { + uint64 numerator; + uint64 denominator; +} fraction; + +static inline fraction +init_fraction(uint64 numerator, uint64 denominator) +{ + return (fraction){ + .numerator = numerator, + .denominator = denominator, + }; +} + +#define zero_fraction \ + ((fraction){ \ + .numerator = 0, \ + .denominator = 1, \ + }) + +static inline fraction +fraction_init_or_zero(uint64 num, uint64 den) +{ + return den ? init_fraction(num, den) : zero_fraction; +} + +// Length of output buffer to snprintf()-into size as string w/ unit specifier +#define SIZE_TO_STR_LEN 20 + +// Format a size value with unit-specifiers, in an output buffer. +char * +size_to_str(char *outbuf, size_t outbuflen, size_t size); + +char * +size_to_fmtstr(char *outbuf, size_t outbuflen, const char *fmtstr, size_t size); + +/* + * Convenience caller macros to convert 'sz' bytes to return a string, + * formatting the input size as human-readable value with unit-specifiers. + */ +// char *size_str(size_t sz) +#define size_str(sz) \ + (({ \ + struct { \ + char buffer[SIZE_TO_STR_LEN]; \ + } onstack_chartmp; \ + size_to_str( \ + onstack_chartmp.buffer, sizeof(onstack_chartmp.buffer), sz); \ + onstack_chartmp; \ + }).buffer) + +// char *size_fmtstr(const char *fmtstr, size_t sz) +#define size_fmtstr(fmtstr, sz) \ + (({ \ + struct { \ + char buffer[SIZE_TO_STR_LEN]; \ + } onstack_chartmp; \ + size_to_fmtstr( \ + onstack_chartmp.buffer, sizeof(onstack_chartmp.buffer), fmtstr, sz); \ + onstack_chartmp; \ + }).buffer) diff --git a/src/platform_linux/poison.h b/src/platform_linux/poison.h index dd8648e65..6072cf5fb 100644 --- a/src/platform_linux/poison.h +++ b/src/platform_linux/poison.h @@ -1,4 +1,4 @@ -// Copyright 2018-2021 VMware, Inc. +// Copyright 2018-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* ********************************************************** @@ -69,18 +69,18 @@ * prama GCC poison does not allow poisoning existing macros. */ #if 0 // Cannot poison existing macros -# pragma GCC poison alloca -# pragma GCC poison assert +# pragma GCC poison alloca +# pragma GCC poison assert # pragma GCC poison EINVAL ENOMEM EINVAL ENOSPC ETIMEDOUT -# pragma GCC poison pthread_cleanup_pop -# pragma GCC poison pthread_cleanup_push +# pragma GCC poison pthread_cleanup_pop +# pragma GCC poison pthread_cleanup_push // macros on some systems. -# pragma GCC poison strncat -# pragma GCC poison strcmp +# pragma GCC poison strncat +# pragma GCC poison strcmp #endif // Cannot poison existing macros -#pragma GCC poison __thread +#pragma GCC poison __thread #pragma GCC poison laio_handle #pragma GCC poison mmap #pragma GCC poison pthread_attr_destroy diff --git a/src/platform_linux/shmem.c b/src/platform_linux/shmem.c index a76f8c162..504967ed9 100644 --- a/src/platform_linux/shmem.c +++ b/src/platform_linux/shmem.c @@ -1,4 +1,4 @@ -// Copyright 2018-2023 VMware, Inc. +// Copyright 2018-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* @@ -7,9 +7,18 @@ * This file contains the implementation for managing shared memory created * for use by SplinterDB and all its innards. */ -#include "platform.h" #include "shmem.h" -#include "util.h" +#include "splinterdb/platform_linux/public_platform.h" +#include "platform_util.h" +#include "platform_machine.h" +#include "platform_threads.h" +#include "platform_log.h" +#include "platform_heap.h" +#include "platform_assert.h" +#include "platform_status.h" +#include "platform_spinlock.h" +#include "platform_units.h" +#include // SplinterDB's shared segment magic identifier. Mainly needed for diagnostics. #define SPLINTERDB_SHMEM_MAGIC (uint64)0x543e4a6d @@ -113,7 +122,7 @@ typedef struct shminfo_usage_stats { * done when freeing any fragment, to see if it's a large-fragment. * ----------------------------------------------------------------------------- */ -typedef struct shmem_heap { +struct shmem_heap { void *shm_start; // Points to start address of shared segment. void *shm_end; // Points to end address; one past end of sh segment void *shm_next; // Points to next 'free' address to allocate from. @@ -131,7 +140,7 @@ typedef struct shmem_heap { uint64 shm_magic; // Magic identifier for shared memory segment int shm_id; // Shared memory ID returned by shmget() -} PLATFORM_CACHELINE_ALIGNED shmem_heap; +} PLATFORM_CACHELINE_ALIGNED; /* Permissions for accessing shared memory and IPC objects */ #define PLATFORM_IPC_OBJS_PERMS (S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP) @@ -189,24 +198,6 @@ platform_heap_id_to_shmaddr(platform_heap_id hid) return (shmem_heap *)hid; } -/* Evaluates to valid 'low' address within shared segment. */ -static inline void * -platform_shm_lop(platform_heap_id hid) -{ - return (void *)platform_heap_id_to_shmaddr(hid); -} - -/* - * Evaluates to valid 'high' address within shared segment. - * shm_end points to one-byte-just-past-end of shared segment. - * Valid 'high' byte is last byte just before shm_end inside segment. - */ -static inline void * -platform_shm_hip(platform_heap_id hid) -{ - return (void *)(platform_heap_id_to_shmaddr(hid)->shm_end - 1); -} - static inline void shm_lock_mem(shmem_heap *shm) { @@ -249,10 +240,9 @@ platform_valid_addr_in_shm(shmem_heap *shmaddr, const void *addr) * region. */ bool -platform_valid_addr_in_heap(platform_heap_id heap_id, const void *addr) +platform_valid_addr_in_heap(shmem_heap *heap, const void *addr) { - return platform_valid_addr_in_shm(platform_heap_id_to_shmaddr(heap_id), - addr); + return platform_valid_addr_in_shm(heap, addr); } /* @@ -360,10 +350,9 @@ platform_shm_print_usage(platform_heap_id hid) * ----------------------------------------------------------------------------- */ platform_status -platform_shmcreate(size_t size, - platform_heap_id *heap_id) // Out +platform_shmcreate(size_t size, shmem_heap **heap) // Out { - platform_assert((*heap_id == NULL), + platform_assert((*heap == NULL), "Heap handle is expected to be NULL while creating a new " "shared segment.\n"); @@ -403,17 +392,15 @@ platform_shmcreate(size_t size, shm->usage.total_bytes = size; shm->usage.free_bytes = free_bytes; - // Return 'heap-ID' handle pointing to start addr of shared segment. - if (heap_id) { - *heap_id = (platform_heap_id *)shmaddr; + // Return heap handle pointing to start addr of shared segment. + if (heap) { + *heap = shm; } - platform_spinlock_init( - &shm->shm_mem_lock, platform_get_module_id(), *heap_id); + platform_spinlock_init(&shm->shm_mem_lock); // Initialize spinlock needed to access memory fragments tracker - platform_spinlock_init( - &shm->shm_mem_frags_lock, platform_get_module_id(), *heap_id); + platform_spinlock_init(&shm->shm_mem_frags_lock); // Always trace creation of shared memory segment. platform_default_log("Completed setup of shared memory of size " @@ -434,21 +421,21 @@ platform_shmcreate(size_t size, * ----------------------------------------------------------------------------- */ void -platform_shmdestroy(platform_heap_id *hid_out) +platform_shmdestroy(shmem_heap **heap) { - if (!hid_out) { + if (!heap) { platform_error_log( - "Error! Attempt to destroy shared memory with NULL heap ID!"); + "Error! Attempt to destroy shared memory with NULL heap pointer!"); return; } - const void *shmaddr = (const void *)platform_heap_id_to_shmaddr(*hid_out); + const void *shmaddr = (const void *)(*heap); - // Heap ID may be coming from the shared segment, itself, that we will + // Heap pointer may be coming from the shared segment, itself, that we will // be detaching from now and freeing, below. So, an attempt to NULL out // this handle after memory is freed will run into an exception. Clear // out this handle prior to all this circus. - *hid_out = NULL; + *heap = NULL; // Use a cached copy in case we are dealing with bogus input shmem address. shmem_heap shmem_heap_struct; @@ -457,12 +444,12 @@ platform_shmdestroy(platform_heap_id *hid_out) shmem_heap *shm = &shmem_heap_struct; if (shm->shm_magic != SPLINTERDB_SHMEM_MAGIC) { - platform_error_log("%s(): Input heap ID, %p, does not seem to be a " + platform_error_log("%s(): Input heap, %p, does not seem to be a " "valid SplinterDB shared segment's start address." " Found magic 0x%lX does not match expected" " magic 0x%lX.\n", __func__, - hid_out, + shmaddr, shm->shm_magic, SPLINTERDB_SHMEM_MAGIC); return; @@ -523,18 +510,18 @@ platform_shmdestroy(platform_heap_id *hid_out) */ // RESOLVE: Pass down user requested alignment and handle it here. void * -platform_shm_alloc(platform_heap_id hid, - const size_t size, - const char *objname, - const char *func, - const char *file, - const int lineno) +platform_shm_alloc(shmem_heap *heap, + const size_t size, + const char *objname, + const char *func, + const char *file, + const int lineno) { - shmem_heap *shm = platform_heap_id_to_shmaddr(hid); + shmem_heap *shm = heap; debug_assert((platform_shm_heap_valid(shm) == TRUE), - "Shared memory heap ID at %p is not a valid shared memory ptr.", - hid); + "Shared memory heap at %p is not a valid shared memory ptr.", + heap); debug_assert(((size % PLATFORM_CACHELINE_SIZE) == 0), "size=%lu is not aligned to PLATFORM_CACHELINE_SIZE", @@ -630,13 +617,13 @@ platform_shm_alloc(platform_heap_id hid, * ----------------------------------------------------------------------------- */ void * -platform_shm_realloc(platform_heap_id hid, - void *oldptr, - const size_t oldsize, - const size_t newsize, - const char *func, - const char *file, - const int lineno) +platform_shm_realloc(shmem_heap *heap, + void *oldptr, + const size_t oldsize, + const size_t newsize, + const char *func, + const char *file, + const int lineno) { debug_assert(((oldptr == NULL) && (oldsize == 0)) || (oldptr && oldsize), "oldptr=%p, oldsize=%lu", @@ -644,22 +631,22 @@ platform_shm_realloc(platform_heap_id hid, oldsize); // We can only realloc from an oldptr that's allocated from shmem - debug_assert(!oldptr || platform_valid_addr_in_heap(hid, oldptr), + debug_assert(!oldptr || platform_valid_addr_in_heap(heap, oldptr), "oldptr=%p is not allocated from shared memory", oldptr); void *retptr = - platform_shm_alloc(hid, newsize, "Unknown", func, file, lineno); + platform_shm_alloc(heap, newsize, "Unknown", func, file, lineno); if (retptr) { // Copy over old contents, if any, and free that memory piece if (oldptr) { memcpy(retptr, oldptr, oldsize); - platform_shm_free(hid, oldptr, "Unknown", func, file, lineno); + platform_shm_free(heap, oldptr, "Unknown", func, file, lineno); } } else { // Report approx memory usage metrics w/o spinlock (diagnostics) - shmem_heap *shm = platform_heap_id_to_shmaddr(hid); + shmem_heap *shm = heap; size_t total_bytes = shm->usage.total_bytes; size_t used_bytes = shm->usage.used_bytes; size_t free_bytes = shm->usage.free_bytes; @@ -705,21 +692,21 @@ platform_shm_realloc(platform_heap_id hid, * ----------------------------------------------------------------------------- */ void -platform_shm_free(platform_heap_id hid, - void *ptr, - const char *objname, - const char *func, - const char *file, - const int lineno) +platform_shm_free(shmem_heap *heap, + void *ptr, + const char *objname, + const char *func, + const char *file, + const int lineno) { - shmem_heap *shm = platform_heap_id_to_shmaddr(hid); + shmem_heap *shm = heap; debug_assert( (platform_shm_heap_valid(shm) == TRUE), - "Shared memory heap ID at %p is not a valid shared memory handle.", - hid); + "Shared memory heap at %p is not a valid shared memory handle.", + heap); - if (!platform_valid_addr_in_heap(hid, ptr)) { + if (!platform_valid_addr_in_heap(heap, ptr)) { platform_error_log("[%s:%d::%s()] -> %s: Requesting to free memory" " at %p, for object '%s' which is a memory chunk not" " allocated from shared memory {start=%p, end=%p}.\n", @@ -729,8 +716,8 @@ platform_shm_free(platform_heap_id hid, __func__, ptr, objname, - platform_shm_lop(hid), - platform_shm_hip(hid)); + (void *)shm, + (void *)(shm->shm_end - 1)); return; } @@ -830,7 +817,7 @@ platform_shm_track_large_alloc(shmem_heap *shm, void *addr, size_t size) frag->frag_addr = addr; frag->frag_size = size; - frag->frag_allocated_to_pid = platform_getpid(); + frag->frag_allocated_to_pid = platform_get_os_pid(); frag->frag_allocated_to_tid = platform_get_tid(); // The freed_by_pid/freed_by_tid == 0 means fragment is still allocated. @@ -896,7 +883,7 @@ platform_shm_track_free(shmem_heap *shm, // Mark the fragment as in-use by recording the process/thread that's // doing the free. - frag->frag_freed_by_pid = platform_getpid(); + frag->frag_freed_by_pid = platform_get_os_pid(); frag->frag_freed_by_tid = platform_get_tid(); if (trace_shmem) { @@ -920,7 +907,7 @@ platform_shm_track_free(shmem_heap *shm, if (!found_tracked_frag && trace_shmem) { platform_default_log("[OS-pid=%d, ThreadID=%lu, %s:%d::%s()] " ", Fragment %p for object '%s' is not tracked\n", - platform_getpid(), + platform_get_os_pid(), platform_get_tid(), file, lineno, @@ -986,7 +973,7 @@ platform_shm_find_large(shmem_heap *shm, found_at_fctr = fctr; // Record the process/thread to which free fragment is being allocated - frag->frag_allocated_to_pid = platform_getpid(); + frag->frag_allocated_to_pid = platform_get_os_pid(); frag->frag_allocated_to_tid = platform_get_tid(); shm->usage.nlarge_frags_inuse++; @@ -1168,11 +1155,10 @@ platform_shm_set_splinterdb_handle(platform_heap_id heap_id, void *addr) } void * -platform_heap_get_splinterdb_handle(const platform_heap_id heap_id) +platform_heap_get_splinterdb_handle(shmem_heap *heap) { - debug_assert(platform_shm_heap_id_valid(heap_id)); - shmem_heap *shm = platform_heap_id_to_shmaddr(heap_id); - return shm->shm_splinterdb_handle; + debug_assert(platform_shm_heap_valid(heap)); + return heap->shm_splinterdb_handle; } /* @@ -1258,29 +1244,29 @@ platform_shm_ctrlblock_size() } /* - * Shmem-accessor interfaces by heap_id. + * Shmem-accessor interfaces by heap. */ size_t -platform_shmsize(platform_heap_id heap_id) +platform_shmsize(shmem_heap *heap) { - return (platform_heap_id_to_shmaddr(heap_id)->usage.total_bytes); + return (heap->usage.total_bytes); } size_t -platform_shmbytes_used(platform_heap_id heap_id) +platform_shmbytes_used(shmem_heap *heap) { - return (platform_heap_id_to_shmaddr(heap_id)->usage.used_bytes); + return (heap->usage.used_bytes); } size_t -platform_shmbytes_free(platform_heap_id heap_id) +platform_shmbytes_free(shmem_heap *heap) { - return (platform_heap_id_to_shmaddr(heap_id)->usage.free_bytes); + return (heap->usage.free_bytes); } void * -platform_shm_next_free_addr(platform_heap_id heap_id) +platform_shm_next_free_addr(shmem_heap *heap) { - return (platform_heap_id_to_shmaddr(heap_id)->shm_next); + return (heap->shm_next); } static void @@ -1297,7 +1283,7 @@ platform_shm_trace_allocs(shmem_heap *shm, "-> %s: %s size=%lu bytes (%s)" " for object '%s', at %p, " "free bytes=%lu (%s).\n", - platform_getpid(), + platform_get_os_pid(), platform_get_tid(), file, lineno, diff --git a/src/platform_linux/shmem.h b/src/platform_linux/shmem.h index c9e339246..5fe5a71e8 100644 --- a/src/platform_linux/shmem.h +++ b/src/platform_linux/shmem.h @@ -1,38 +1,41 @@ -// Copyright 2018-2023 VMware, Inc. +// Copyright 2018-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 #pragma once +#include "platform_status.h" #include #include +typedef struct shmem_heap shmem_heap; + platform_status -platform_shmcreate(size_t size, platform_heap_id *heap_id); +platform_shmcreate(size_t size, shmem_heap **heap); void -platform_shmdestroy(platform_heap_id *heap_id); +platform_shmdestroy(shmem_heap **heap); /* * Allocate memory fragment from the shared memory of requested 'size'. */ void * -platform_shm_alloc(platform_heap_id hid, - const size_t size, - const char *objname, - const char *func, - const char *file, - const int lineno); +platform_shm_alloc(shmem_heap *heap, + const size_t size, + const char *objname, + const char *func, + const char *file, + const int lineno); /* * Free the memory fragment at 'ptr' address. */ void -platform_shm_free(platform_heap_id hid, - void *ptr, - const char *objname, - const char *func, - const char *file, - const int lineno); +platform_shm_free(shmem_heap *heap, + void *ptr, + const char *objname, + const char *func, + const char *file, + const int lineno); /* * Reallocate the memory (fragment) at 'oldptr' of size 'oldsize' bytes. @@ -44,13 +47,13 @@ platform_shm_free(platform_heap_id hid, * Returns ptr to re-allocated memory of 'newsize' bytes. */ void * -platform_shm_realloc(platform_heap_id hid, - void *oldptr, - const size_t oldsize, - const size_t newsize, - const char *func, - const char *file, - const int lineno); +platform_shm_realloc(shmem_heap *heap, + void *oldptr, + const size_t oldsize, + const size_t newsize, + const char *func, + const char *file, + const int lineno); void platform_shm_tracing_init(const bool trace_shmem, @@ -85,23 +88,22 @@ size_t platform_shm_ctrlblock_size(); /* - * Interfaces to retrieve size(s) using heap_id, which is what's - * known externally to memory allocation interfaces. + * Interfaces to retrieve size(s) using heap. */ size_t -platform_shmsize(platform_heap_id heap_id); +platform_shmsize(shmem_heap *heap); size_t -platform_shmbytes_free(platform_heap_id heap_id); +platform_shmbytes_free(shmem_heap *heap); size_t -platform_shmbytes_used(platform_heap_id heap_id); +platform_shmbytes_used(shmem_heap *heap); void * -platform_shm_next_free_addr(platform_heap_id heap_id); +platform_shm_next_free_addr(shmem_heap *heap); bool -platform_valid_addr_in_heap(platform_heap_id heap_id, const void *addr); +platform_valid_addr_in_heap(shmem_heap *heap, const void *addr); void * -platform_heap_get_splinterdb_handle(platform_heap_id heap_id); +platform_heap_get_splinterdb_handle(shmem_heap *heap); diff --git a/src/rc_allocator.c b/src/rc_allocator.c index 50159d0b3..a9b97e958 100644 --- a/src/rc_allocator.c +++ b/src/rc_allocator.c @@ -1,4 +1,4 @@ -// Copyright 2018-2021 VMware, Inc. +// Copyright 2018-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* @@ -9,11 +9,12 @@ *------------------------------------------------------------------------------ */ -#include "platform.h" - #include "rc_allocator.h" -#include "io.h" - +#include "platform_io.h" +#include "platform_hash.h" +#include "platform_buffer.h" +#include "platform_mutex.h" +#include "platform_typed_alloc.h" #include "poison.h" #define RC_ALLOCATOR_META_PAGE_CSUM_SEED (2718281828) @@ -269,7 +270,7 @@ platform_status rc_allocator_valid_config(allocator_config *cfg) { platform_status rc = STATUS_OK; - rc = laio_config_valid(cfg->io_cfg); + rc = io_config_valid(cfg->io_cfg); if (!SUCCESS(rc)) { return rc; } diff --git a/src/rc_allocator.h b/src/rc_allocator.h index 6c85cdaa7..c41c1471f 100644 --- a/src/rc_allocator.h +++ b/src/rc_allocator.h @@ -1,4 +1,4 @@ -// Copyright 2018-2021 VMware, Inc. +// Copyright 2018-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* @@ -9,8 +9,10 @@ #pragma once +#include "platform_hash.h" +#include "platform_buffer.h" +#include "platform_mutex.h" #include "allocator.h" -#include "platform.h" #include "util.h" /* diff --git a/src/routing_filter.c b/src/routing_filter.c index 4bc38ad72..49e6b502e 100644 --- a/src/routing_filter.c +++ b/src/routing_filter.c @@ -1,4 +1,4 @@ -// Copyright 2018-2021 VMware, Inc. +// Copyright 2018-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* @@ -8,14 +8,17 @@ * This file contains the implementation for a routing filter *---------------------------------------------------------------------- */ -#include "platform.h" #include "routing_filter.h" #include "PackedArray.h" #include "mini_allocator.h" #include "iterator.h" -#include -#include -#include +#include "platform_hash.h" +#include "platform_typed_alloc.h" +#include "platform_assert.h" +#include "platform_threads.h" + +// This is used only for log(), which we should figure out how to avoid using, +// so I'm not going to bother to platformize it. #include #include "poison.h" @@ -108,7 +111,7 @@ RadixSort(uint32 *pData, platform_assert((mIndex[j][c] < count), "OS-pid=%d, thread-ID=%lu, i=%u, j=%u, c=%d" ", mIndex[j][c]=%d, count=%u pData=%p pTemp=%p\n", - platform_getpid(), + platform_get_os_pid(), platform_get_tid(), i, j, @@ -431,7 +434,7 @@ routing_filter_add(cache *cc, filter->meta_head = meta_head; // filters use an unkeyed mini allocator mini_allocator mini; - mini_init(&mini, cc, NULL, filter->meta_head, 0, 1, PAGE_TYPE_FILTER); + mini_init(&mini, cc, filter->meta_head, 0, 1, PAGE_TYPE_FILTER); // set up the index pages uint64 addrs_per_page = page_size / sizeof(uint64); @@ -708,8 +711,9 @@ routing_filter_estimate_unique_fp(cache *cc, for (uint64 i = 0; i != num_filters; i++) { total_num_fp += filter[i].num_fingerprints; } - uint32 buffer_size = total_num_fp / 12; - uint32 alloc_size = buffer_size + cfg->index_size; + uint32 buffer_size = total_num_fp / 12; + uint32 alloc_size = buffer_size + cfg->index_size; + // NOLINTNEXTLINE(bugprone-sizeof-expression) uint32 *local = TYPED_ARRAY_ZALLOC(hid, local, alloc_size * sizeof(uint32)); uint32 *fp_arr = local; uint32 *count = local + buffer_size; @@ -1110,17 +1114,20 @@ uint32 routing_filter_estimate_unique_keys_from_count(const routing_config *cfg, uint64 num_unique) { - double universe_size = 1UL << cfg->fingerprint_size; - double unseen_fp = universe_size - num_unique; + double universe_size = 1UL << cfg->fingerprint_size; + double unseen_fp = universe_size - num_unique; + double universe_size_2 = universe_size * universe_size; + double universe_size_4 = universe_size_2 * universe_size_2; + double unseen_fp_2 = unseen_fp * unseen_fp; + double unseen_fp_4 = unseen_fp_2 * unseen_fp_2; /* * Compute the difference H_|U| - H_{|U| - #unique_fp}, where U is the fp * universe. */ - double harmonic_diff = - log(universe_size) - log(unseen_fp) - + 1 / 2.0 * (1 / universe_size - 1 / unseen_fp) - - 1 / 12.0 * (1 / pow(universe_size, 2) - 1 / pow(unseen_fp, 2)) - + 1 / 120.0 * (1 / pow(universe_size, 4) - 1 / pow(unseen_fp, 4)); + double harmonic_diff = log(universe_size) - log(unseen_fp) + + 1 / 2.0 * (1 / universe_size - 1 / unseen_fp) + - 1 / 12.0 * (1 / universe_size_2 - 1 / unseen_fp_2) + + 1 / 120.0 * (1 / universe_size_4 - 1 / unseen_fp_4); uint32 estimated_input_keys = universe_size * harmonic_diff; return estimated_input_keys; } diff --git a/src/routing_filter.h b/src/routing_filter.h index 47a96cde2..70366c8b6 100644 --- a/src/routing_filter.h +++ b/src/routing_filter.h @@ -1,4 +1,4 @@ -// Copyright 2018-2021 VMware, Inc. +// Copyright 2018-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* @@ -13,7 +13,7 @@ #include "iterator.h" #include "splinterdb/data.h" #include "util.h" -#include "platform.h" +#include "platform_hash.h" /* * In Splinter, there is a strict max number of compacted tuples in a node, so diff --git a/src/shard_log.c b/src/shard_log.c index 309fccda8..6b2bae23e 100644 --- a/src/shard_log.c +++ b/src/shard_log.c @@ -1,4 +1,4 @@ -// Copyright 2018-2021 VMware, Inc. +// Copyright 2018-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* @@ -9,11 +9,14 @@ *----------------------------------------------------------------------------- */ -#include "platform.h" - #include "shard_log.h" #include "data_internal.h" - +#include "platform_sleep.h" +#include "platform_hash.h" +#include "platform_typed_alloc.h" +#include "platform_assert.h" +#include "platform_threads.h" +#include "platform_sort.h" #include "poison.h" #define SHARD_WAIT 1 @@ -109,8 +112,7 @@ shard_log_init(shard_log *log, cache *cc, shard_log_config *cfg) thread_data->offset = 0; } - log->addr = mini_init( - &log->mini, cc, log->cfg->data_cfg, log->meta_head, 0, 1, PAGE_TYPE_LOG); + log->addr = mini_init(&log->mini, cc, log->meta_head, 0, 1, PAGE_TYPE_LOG); // platform_default_log("addr: %lu meta_head: %lu\n", log->addr, // log->meta_head); @@ -142,7 +144,7 @@ struct ONDISK log_entry { ondisk_tuple tuple; }; -#define INVALID_GENERATION ((uint64)-1) +#define INVALID_GENERATION ((uint64) - 1) static key log_entry_key(log_entry *le) diff --git a/src/shard_log.h b/src/shard_log.h index 980c291a9..0129e9d2b 100644 --- a/src/shard_log.h +++ b/src/shard_log.h @@ -1,4 +1,4 @@ -// Copyright 2018-2021 VMware, Inc. +// Copyright 2018-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* @@ -10,6 +10,8 @@ #pragma once #include "log.h" +#include "platform_threads.h" +#include "platform_hash.h" #include "cache.h" #include "iterator.h" #include "splinterdb/data.h" diff --git a/src/splinterdb.c b/src/splinterdb.c index b5482eb39..efa2ea542 100644 --- a/src/splinterdb.c +++ b/src/splinterdb.c @@ -1,4 +1,4 @@ -// Copyright 2021 VMware, Inc. +// Copyright 2021-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* @@ -14,14 +14,14 @@ */ #include "splinterdb/splinterdb.h" -#include "platform.h" #include "clockcache.h" -#include "platform_linux/platform.h" #include "rc_allocator.h" #include "core.h" -#include "btree_private.h" #include "shard_log.h" #include "splinterdb_tests_private.h" +#include "platform_typed_alloc.h" +#include "platform_assert.h" +#include "platform_units.h" #include "poison.h" const char *BUILD_VERSION = "splinterdb_build_version " GIT_VERSION; @@ -40,7 +40,7 @@ splinterdb_get_version() typedef struct splinterdb { task_system *task_sys; io_config io_cfg; - platform_io_handle io_handle; + io_handle *io_handle; allocator_config allocator_cfg; rc_allocator allocator_handle; clockcache_config cache_cfg; @@ -75,20 +75,20 @@ static void splinterdb_config_set_defaults(splinterdb_config *cfg) { if (!cfg->page_size) { - cfg->page_size = LAIO_DEFAULT_PAGE_SIZE; + cfg->page_size = IO_DEFAULT_PAGE_SIZE; } if (!cfg->extent_size) { - cfg->extent_size = LAIO_DEFAULT_EXTENT_SIZE; + cfg->extent_size = IO_DEFAULT_EXTENT_SIZE; } if (!cfg->io_flags) { - cfg->io_flags = O_RDWR | O_CREAT; + cfg->io_flags = IO_DEFAULT_FLAGS; } if (!cfg->io_perms) { - cfg->io_perms = 0755; + cfg->io_perms = IO_DEFAULT_PERMS; } if (!cfg->io_async_queue_depth) { - cfg->io_async_queue_depth = 256; + cfg->io_async_queue_depth = IO_DEFAULT_ASYNC_QUEUE_DEPTH; } if (!cfg->btree_rough_count_height) { @@ -176,7 +176,7 @@ splinterdb_init_config(const splinterdb_config *kvs_cfg, // IN cfg.filename); // Validate IO-configuration parameters - rc = laio_config_valid(&kvs->io_cfg); + rc = io_config_valid(&kvs->io_cfg); if (!SUCCESS(rc)) { return rc; } @@ -195,8 +195,7 @@ splinterdb_init_config(const splinterdb_config *kvs_cfg, // IN num_bg_threads[TASK_TYPE_MEMTABLE] = kvs_cfg->num_memtable_bg_threads; num_bg_threads[TASK_TYPE_NORMAL] = kvs_cfg->num_normal_bg_threads; - rc = task_system_config_init( - &kvs->task_cfg, cfg.use_stats, num_bg_threads, core_get_scratch_size()); + rc = task_system_config_init(&kvs->task_cfg, cfg.use_stats, num_bg_threads); if (!SUCCESS(rc)) { return rc; } @@ -305,15 +304,14 @@ splinterdb_create_or_open(const splinterdb_config *kvs_cfg, // IN platform_shm_set_splinterdb_handle(use_this_heap_id, (void *)kvs); } - status = io_handle_init(&kvs->io_handle, &kvs->io_cfg, kvs->heap_id); - if (!SUCCESS(status)) { - platform_error_log("Failed to initialize IO handle: %s\n", - platform_status_to_string(status)); - goto io_handle_init_failed; + kvs->io_handle = io_handle_create(&kvs->io_cfg, kvs->heap_id); + if (kvs->io_handle == NULL) { + platform_error_log("Failed to create IO handle\n"); + status = STATUS_NO_MEMORY; + goto io_handle_create_failed; } - status = task_system_create( - kvs->heap_id, &kvs->io_handle, &kvs->task_sys, &kvs->task_cfg); + status = task_system_create(kvs->heap_id, &kvs->task_sys, &kvs->task_cfg); if (!SUCCESS(status)) { platform_error_log( "Failed to initialize SplinterDB task system state: %s\n", @@ -324,13 +322,13 @@ splinterdb_create_or_open(const splinterdb_config *kvs_cfg, // IN if (open_existing) { status = rc_allocator_mount(&kvs->allocator_handle, &kvs->allocator_cfg, - (io_handle *)&kvs->io_handle, + kvs->io_handle, kvs->heap_id, platform_get_module_id()); } else { status = rc_allocator_init(&kvs->allocator_handle, &kvs->allocator_cfg, - (io_handle *)&kvs->io_handle, + kvs->io_handle, kvs->heap_id, platform_get_module_id()); } @@ -343,7 +341,7 @@ splinterdb_create_or_open(const splinterdb_config *kvs_cfg, // IN status = clockcache_init(&kvs->cache_handle, &kvs->cache_cfg, - (io_handle *)&kvs->io_handle, + kvs->io_handle, (allocator *)&kvs->allocator_handle, "splinterdb", kvs->heap_id, @@ -389,8 +387,8 @@ splinterdb_create_or_open(const splinterdb_config *kvs_cfg, // IN deinit_system: task_system_destroy(kvs->heap_id, &kvs->task_sys); deinit_iohandle: - io_handle_deinit(&kvs->io_handle); -io_handle_init_failed: + io_handle_destroy(kvs->io_handle); +io_handle_create_failed: deinit_kvhandle: // Depending on the place where a configuration / setup error lead // us to here via a 'goto', heap_id handle, if in use, may be in a @@ -461,7 +459,7 @@ splinterdb_close(splinterdb **kvs_in) // IN clockcache_deinit(&kvs->cache_handle); rc_allocator_unmount(&kvs->allocator_handle); task_system_destroy(kvs->heap_id, &kvs->task_sys); - io_handle_deinit(&kvs->io_handle); + io_handle_destroy(kvs->io_handle); // Free resources carefully to avoid ASAN-test failures platform_heap_id heap_id = kvs->heap_id; @@ -474,58 +472,6 @@ splinterdb_close(splinterdb **kvs_in) // IN } -/* - *----------------------------------------------------------------------------- - * splinterdb_register_thread -- - * - * Allocate scratch space and register the current thread. - * - * Any thread, other than the initializing thread, must call this function - * exactly once before using the splinterdb. - * - * Notes: - * - The task system imposes a limit of MAX_THREADS live at any time - * - * Results: - * None. - * - * Side effects: - * Allocates memory - *----------------------------------------------------------------------------- - */ -void -splinterdb_register_thread(splinterdb *kvs) // IN -{ - platform_assert(kvs != NULL); - - size_t scratch_size = core_get_scratch_size(); - platform_status rc = task_register_this_thread(kvs->task_sys, scratch_size); - platform_assert_status_ok(rc); -} - -/* - *----------------------------------------------------------------------------- - * splinterdb_deregister_thread -- - * - * Free scratch space. - * Call this function before exiting a registered thread. - * Otherwise, you'll leak memory. - * - * Results: - * None. - * - * Side effects: - * Frees memory - *----------------------------------------------------------------------------- - */ -void -splinterdb_deregister_thread(splinterdb *kvs) -{ - platform_assert(kvs != NULL); - - task_deregister_this_thread(kvs->task_sys); -} - /* *----------------------------------------------------------------------------- * splinterdb_insert_raw_message -- @@ -837,10 +783,10 @@ splinterdb_get_task_system_handle(const splinterdb *kvs) return kvs->task_sys; } -const platform_io_handle * +const io_handle * splinterdb_get_io_handle(const splinterdb *kvs) { - return &kvs->io_handle; + return kvs->io_handle; } const allocator * @@ -865,4 +811,4 @@ const memtable_context * splinterdb_get_memtable_context_handle(const splinterdb *kvs) { return kvs->spl->mt_ctxt; -} +} \ No newline at end of file diff --git a/src/splinterdb_tests_private.h b/src/splinterdb_tests_private.h index 334cc1be2..bc7d2cde3 100644 --- a/src/splinterdb_tests_private.h +++ b/src/splinterdb_tests_private.h @@ -1,4 +1,4 @@ -// Copyright 2023 VMware, Inc. +// Copyright 2023-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* @@ -27,7 +27,7 @@ splinterdb_get_heap_id(const splinterdb *kvs); const task_system * splinterdb_get_task_system_handle(const splinterdb *kvs); -const platform_io_handle * +const io_handle * splinterdb_get_io_handle(const splinterdb *kvs); const allocator * diff --git a/src/task.c b/src/task.c index aabfdb980..c55b1818a 100644 --- a/src/task.c +++ b/src/task.c @@ -1,11 +1,11 @@ -// Copyright 2018-2021 VMware, Inc. +// Copyright 2018-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 -#include "platform.h" #include "task.h" -#include "util.h" -#include "io.h" - +#include "platform_log.h" +#include "platform_time.h" +#include "platform_typed_alloc.h" +#include "platform_sleep.h" #include "poison.h" const char *task_type_name[] = {"TASK_TYPE_INVALID", @@ -14,442 +14,6 @@ const char *task_type_name[] = {"TASK_TYPE_INVALID", _Static_assert((ARRAY_SIZE(task_type_name) == NUM_TASK_TYPES), "Array task_type_name[] is incorrectly sized."); -/**************************************** - * Thread ID allocation and management * - ****************************************/ -/* - * task_init_tid_bitmask() - Initialize the global bitmask of active threads in - * the task system structure to indicate that no threads are currently active. - */ -static void -task_init_tid_bitmask(uint64 *tid_bitmask) -{ - /* - * This is a special bitmask where 1 indicates free and 0 indicates - * allocated. So, we set all bits to 1 during init. - * We set all bits to 1 even in the unused portion of the last bitmask - * to maintain compatibility with the cleanup assertion. - */ - int num_bitmasks = (MAX_THREADS + 63) / 64; - - for (int i = 0; i < num_bitmasks; i++) { - tid_bitmask[i] = (uint64)-1; - } -} - -static inline uint64 * -task_system_get_tid_bitmask(task_system *ts) -{ - return ts->tid_bitmask; -} - -static threadid * -task_system_get_max_tid(task_system *ts) -{ - return &ts->max_tid; -} - -/* - * Allocate a threadid. Returns INVALID_TID when no tid is available. - */ -static threadid -task_allocate_threadid(task_system *ts) -{ - threadid tid = INVALID_TID; - uint64 *tid_bitmask = task_system_get_tid_bitmask(ts); - uint64 old_bitmask; - uint64 new_bitmask; - - while (__sync_lock_test_and_set(&ts->tid_bitmask_lock, 1)) { - // spin - } - - int i; - uint64 pos = 0; - for (i = 0; pos == 0 && i < (MAX_THREADS + 63) / 64; i++) { - old_bitmask = tid_bitmask[i]; - // first bit set to 1 starting from LSB. - pos = __builtin_ffsl(old_bitmask); - } - - if (pos == 0) { - __sync_lock_release(&ts->tid_bitmask_lock); - platform_error_log("No thread id available"); - return INVALID_TID; - } - - i--; - - // builtin_ffsl returns the position plus 1. - tid = pos - 1; - // set bit at that position to 0, indicating in use. - new_bitmask = (old_bitmask & ~(1ULL << tid)); - int r = - __sync_bool_compare_and_swap(&tid_bitmask[i], old_bitmask, new_bitmask); - platform_assert(r); - - __sync_lock_release(&ts->tid_bitmask_lock); - - tid += i * 64; - - // Invariant: we have successfully allocated tid - - // atomically update the max_tid. - threadid *max_tid = task_system_get_max_tid(ts); - threadid tmp = *max_tid; - while (tmp < tid && !__sync_bool_compare_and_swap(max_tid, tmp, tid)) { - tmp = *max_tid; - } - - return tid; -} - -/* - * De-registering a task frees up the thread's index so that it can be re-used. - */ -static void -task_deallocate_threadid(task_system *ts, threadid tid) -{ - uint64 *tid_bitmask = task_system_get_tid_bitmask(ts); - - while (__sync_lock_test_and_set(&ts->tid_bitmask_lock, 1)) { - // spin - } - - uint64 bitmask_val = tid_bitmask[tid / 64]; - - // Ensure that caller is only clearing for a thread that's in-use. - platform_assert(!(bitmask_val & (1ULL << (tid % 64))), - "Thread [%lu] is expected to be in-use. Bitmap: 0x%lx", - tid, - bitmask_val); - - // set bit back to 1 to indicate a free slot. - uint64 tmp_bitmask = tid_bitmask[tid / 64]; - uint64 new_value = tmp_bitmask | (1ULL << (tid % 64)); - while (!__sync_bool_compare_and_swap( - tid_bitmask + tid / 64, tmp_bitmask, new_value)) - { - tmp_bitmask = tid_bitmask[tid / 64]; - new_value = tmp_bitmask | (1ULL << (tid % 64)); - } - - __sync_lock_release(&ts->tid_bitmask_lock); -} - - -/* - * Return the max thread-index across all active tasks. - * Mainly intended as a testing hook. - */ -threadid -task_get_max_tid(task_system *ts) -{ - threadid *max_tid = task_system_get_max_tid(ts); - /* - * The caller(s) iterate till < return_value from this function. - * We have assigned threads till max_tid, hence return plus - * one to ensure that the last thread is covered in the iteration. - */ - return *max_tid + 1; -} - -/**************************************** - * Thread creation hooks * - ****************************************/ - -static platform_status -task_register_hook(task_system *ts, task_hook newhook) -{ - int my_hook_idx = __sync_fetch_and_add(&ts->num_hooks, 1); - if (my_hook_idx >= TASK_MAX_HOOKS) { - return STATUS_LIMIT_EXCEEDED; - } - ts->hooks[my_hook_idx] = newhook; - - return STATUS_OK; -} - -static void -task_system_io_register_thread(task_system *ts) -{ - io_register_thread(&ts->ioh->super); -} - -static void -task_system_io_deregister_thread(task_system *ts) -{ - io_deregister_thread(&ts->ioh->super); -} - -/* - * This is part of task initialization and needs to be called at the - * beginning of the main thread that uses the task, similar to how - * __attribute__((constructor)) works. - */ -static void -register_standard_hooks(task_system *ts) -{ - // hooks need to be initialized only once. - if (__sync_fetch_and_add(&ts->hook_init_done, 1) == 0) { - task_register_hook(ts, task_system_io_register_thread); - } -} - -static void -task_run_thread_hooks(task_system *ts) -{ - for (int i = 0; i < ts->num_hooks; i++) { - ts->hooks[i](ts); - } -} - -/**************************************** - * Thread creation * - ****************************************/ - -typedef struct { - platform_thread_worker func; - void *arg; - - task_system *ts; - threadid tid; - platform_heap_id heap_id; -} thread_invoke; - -/* - * task_invoke_with_hooks() - Single interface to invoke a user-specified - * call-back function, 'func', to perform Splinter work. Both user-threads' - * and background-threads' creation goes through this interface. - * - * A thread has been created with this function as the worker function. Also, - * the thread-creator has registered another call-back function to execute. - * This function invokes that call-back function bracketted by calls to - * register / deregister the thread with Splinter's task system. This allows - * the call-back function 'func' to now call Splinter APIs to do diff things. - */ -static void -task_invoke_with_hooks(void *func_and_args) -{ - thread_invoke *thread_started = (thread_invoke *)func_and_args; - platform_thread_worker func = thread_started->func; - void *arg = thread_started->arg; - - platform_assert(thread_started->tid < MAX_THREADS); - platform_set_tid(thread_started->tid); - - task_run_thread_hooks(thread_started->ts); - - // Execute the user-provided call-back function which is where - // the actual Splinter work will be done. - func(arg); - - // Release scratch space, release thread-ID - // For background threads, also, IO-deregistration will happen here. - task_deregister_this_thread(thread_started->ts); - - platform_free(thread_started->heap_id, func_and_args); -} - -/* - * task_create_thread_with_hooks() - Creates a thread to execute func - * with argument 'arg'. - */ -static platform_status -task_create_thread_with_hooks(platform_thread *thread, - bool32 detached, - platform_thread_worker func, - void *arg, - size_t scratch_size, - task_system *ts, - platform_heap_id hid) -{ - platform_status ret; - - threadid newtid = task_allocate_threadid(ts); - if (newtid == INVALID_TID) { - platform_error_log("Cannot create a new thread as the limit on" - " concurrent threads, %d, will be exceeded.\n", - MAX_THREADS); - return STATUS_BUSY; - } - - if (0 < scratch_size) { - char *scratch = TYPED_MANUAL_ZALLOC(ts->heap_id, scratch, scratch_size); - if (scratch == NULL) { - ret = STATUS_NO_MEMORY; - goto dealloc_tid; - } - ts->thread_scratch[newtid] = scratch; - } - - thread_invoke *thread_to_create = TYPED_ZALLOC(hid, thread_to_create); - if (thread_to_create == NULL) { - ret = STATUS_NO_MEMORY; - goto free_scratch; - } - - thread_to_create->func = func; - thread_to_create->arg = arg; - thread_to_create->heap_id = hid; - thread_to_create->ts = ts; - thread_to_create->tid = newtid; - - ret = platform_thread_create( - thread, detached, task_invoke_with_hooks, thread_to_create, hid); - if (!SUCCESS(ret)) { - goto free_thread; - } - - return ret; - -free_thread: - platform_free(hid, thread_to_create); -free_scratch: - if (ts->thread_scratch[newtid] != NULL) { - platform_free(ts->heap_id, ts->thread_scratch[newtid]); - } -dealloc_tid: - task_deallocate_threadid(ts, newtid); - return ret; -} - -/* - * task_thread_create() - Create a new task and to do the required - * registration with Splinter. - */ -platform_status -task_thread_create(const char *name, - platform_thread_worker func, - void *arg, - size_t scratch_size, - task_system *ts, - platform_heap_id hid, - platform_thread *thread) -{ - platform_thread thr; - platform_status ret; - - ret = task_create_thread_with_hooks( - &thr, FALSE, func, arg, scratch_size, ts, hid); - if (!SUCCESS(ret)) { - platform_error_log("Could not create a thread: %s\n", - platform_status_to_string(ret)); - return ret; - } - - if (thread != NULL) { - *thread = thr; - } - return STATUS_OK; -} - -/****************************************************************************** - * Registering/deregistering threads that were not started by the task - * system. - ******************************************************************************/ - -/* - * task_register_thread(): Register this new thread with the task system. - * - * This is for use by threads created outside of the splinter system. - * - * Registration implies: - * - Acquire a new thread ID (index) for this to-be-active thread - * - Allocate scratch space for this thread, and track this space.. - */ -platform_status -task_register_thread(task_system *ts, - uint64 scratch_size, - const char *file, - const int lineno, - const char *func) -{ - threadid thread_tid; - - thread_tid = platform_get_tid(); - - // Before registration, all SplinterDB threads' tid will be its default - // value; i.e. INVALID_TID. However, for the special case when we run tests - // that simulate process-model of execution, we fork() from the main test. - // Before registration, this 'thread' inherits the thread ID from the main - // thread, which will be == 0. - platform_assert(((thread_tid == INVALID_TID) || (thread_tid == 0)), - "[%s:%d::%s()] Attempt to register thread that is already " - "registered as thread %lu\n", - file, - lineno, - func, - thread_tid); - - thread_tid = task_allocate_threadid(ts); - // Unavailable threads is a temporary state that could go away. - if (thread_tid == INVALID_TID) { - return STATUS_BUSY; - } - - platform_assert(ts->thread_scratch[thread_tid] == NULL, - "Scratch space should not yet exist for tid %lu.", - thread_tid); - - if (0 < scratch_size) { - char *scratch = TYPED_MANUAL_ZALLOC(ts->heap_id, scratch, scratch_size); - if (scratch == NULL) { - platform_default_log("[%s:%d::%s()] Error! Failed to allocate scratch " - "space for thread %lu\n", - file, - lineno, - func, - thread_tid); - task_deallocate_threadid(ts, thread_tid); - return STATUS_NO_MEMORY; - } - ts->thread_scratch[thread_tid] = scratch; - } - - platform_assert(thread_tid < MAX_THREADS); - platform_set_tid(thread_tid); - task_run_thread_hooks(ts); - - return STATUS_OK; -} - -/* - * task_deregister_thread() - Deregister an active thread from the task system. - * - * Deregistration involves: - * - Releasing any scratch space acquired for this thread. - * - De-registering w/ IO sub-system, which will release IO resources - * - Clearing the thread ID (index) for this thread - */ -void -task_deregister_thread(task_system *ts, - const char *file, - const int lineno, - const char *func) -{ - threadid tid = platform_get_tid(); - - platform_assert( - tid != INVALID_TID, - "[%s:%d::%s()] Error! Attempt to deregister unregistered thread.\n", - file, - lineno, - func); - - // Some [test] callers may have created a task w/o requesting for any - // scratch space. So, check before trying to free memory. - void *scratch = ts->thread_scratch[tid]; - if (scratch != NULL) { - platform_free(ts->heap_id, scratch); - ts->thread_scratch[tid] = NULL; - } - - task_system_io_deregister_thread(ts); - platform_set_tid(INVALID_TID); - task_deallocate_threadid(ts, tid); // allow thread id to be re-used -} - /**************************************** * Background task management ****************************************/ @@ -516,8 +80,7 @@ task_group_run_task(task_group *group, task *assigned_task) } } - assigned_task->func(assigned_task->arg, - task_system_get_thread_scratch(group->ts, tid)); + assigned_task->func(assigned_task->arg); if (group->use_stats) { current = platform_timestamp_elapsed(current); @@ -592,7 +155,7 @@ task_group_stop_and_wait_for_threads(task_group *group) // Allow all background threads to wrap up their work. for (uint8 i = 0; i < num_threads; i++) { - platform_thread_join(group->bg.threads[i]); + platform_thread_join(&group->bg.threads[i]); group->bg.num_threads--; } } @@ -608,8 +171,7 @@ static platform_status task_group_init(task_group *group, task_system *ts, bool32 use_stats, - uint8 num_bg_threads, - uint64 scratch_size) + uint8 num_bg_threads) { ZERO_CONTENTS(group); group->ts = ts; @@ -623,13 +185,11 @@ task_group_init(task_group *group, } for (uint8 i = 0; i < num_bg_threads; i++) { - rc = task_thread_create("splinter-bg-thread", - task_worker_thread, - (void *)group, - scratch_size, - ts, - hid, - &group->bg.threads[i]); + rc = platform_thread_create(&group->bg.threads[i], + FALSE, + task_worker_thread, + (void *)group, + ts->heap_id); if (!SUCCESS(rc)) { task_group_stop_and_wait_for_threads(group); goto out; @@ -862,16 +422,14 @@ task_config_valid(const uint64 num_background_threads[NUM_TASK_TYPES]) platform_status task_system_config_init(task_system_config *task_cfg, bool32 use_stats, - const uint64 num_bg_threads[NUM_TASK_TYPES], - uint64 scratch_size) + const uint64 num_bg_threads[NUM_TASK_TYPES]) { platform_status rc = task_config_valid(num_bg_threads); if (!SUCCESS(rc)) { return rc; } - task_cfg->use_stats = use_stats; - task_cfg->scratch_size = scratch_size; + task_cfg->use_stats = use_stats; memcpy(task_cfg->num_background_threads, num_bg_threads, @@ -881,8 +439,7 @@ task_system_config_init(task_system_config *task_cfg, /* * ----------------------------------------------------------------------------- - * Task system initializer. Makes sure that the initial thread has an - * adequately sized scratch space. + * Task system initializer. * * Needs to be called at the beginning of the main thread that uses splinter, * similar to how __attribute__((constructor)) works. @@ -890,7 +447,6 @@ task_system_config_init(task_system_config *task_cfg, */ platform_status task_system_create(platform_heap_id hid, - platform_io_handle *ioh, task_system **system, const task_system_config *cfg) { @@ -904,27 +460,15 @@ task_system_create(platform_heap_id hid, *system = NULL; return STATUS_NO_MEMORY; } - ts->cfg = cfg; - ts->ioh = ioh; - ts->heap_id = hid; - ts->tid_bitmask_lock = 0; - task_init_tid_bitmask(ts->tid_bitmask); - - // task initialization - register_standard_hooks(ts); - - // Ensure that the main thread gets registered and init'ed first before - // any background threads are created. (Those may grab their own tids.). - task_register_this_thread(ts, cfg->scratch_size); + ts->cfg = cfg; + ts->heap_id = hid; for (task_type type = TASK_TYPE_FIRST; type != NUM_TASK_TYPES; type++) { platform_status rc = task_group_init(&ts->group[type], ts, cfg->use_stats, - cfg->num_background_threads[type], - cfg->scratch_size); + cfg->num_background_threads[type]); if (!SUCCESS(rc)) { - task_deregister_this_thread(ts); task_system_destroy(hid, &ts); *system = NULL; return rc; @@ -959,30 +503,10 @@ task_system_destroy(platform_heap_id hid, task_system **ts_in) for (task_type type = TASK_TYPE_FIRST; type != NUM_TASK_TYPES; type++) { task_group_deinit(&ts->group[type]); } - threadid tid = platform_get_tid(); - if (tid != INVALID_TID) { - task_deregister_this_thread(ts); - } - for (int i = 0; i < ARRAY_SIZE(ts->tid_bitmask); i++) { - platform_assert( - ts->tid_bitmask[i] == ((uint64)-1), - "Destroying task system that still has some registered threads." - ", tid=%lu, tid_bitmask[%d] = %lx\n", - tid, - i, - ts->tid_bitmask[i]); - } platform_free(hid, ts); *ts_in = (task_system *)NULL; } -void * -task_system_get_thread_scratch(task_system *ts, const threadid tid) -{ - platform_assert((tid < MAX_THREADS), "tid=%lu", tid); - return ts->thread_scratch[tid]; -} - static void task_group_print_stats(task_group *group, task_type type) { diff --git a/src/task.h b/src/task.h index 65c9a4bfa..8fb9d41b8 100644 --- a/src/task.h +++ b/src/task.h @@ -1,14 +1,42 @@ -// Copyright 2018-2021 VMware, Inc. +// Copyright 2018-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 #pragma once -#include "platform.h" +#include "splinterdb/platform_linux/public_platform.h" +#include "platform_status.h" +#include "platform_heap.h" +#include "platform_threads.h" +#include "platform_condvar.h" + +/* + * We maintain separate task groups for the memtable because memtable + * jobs are much shorter than other jobs and are latency critical. By + * separating them out, we can devote a small number of threads to + * deal with these small, latency critical tasks. + */ +typedef enum task_type { + TASK_TYPE_INVALID = 0, + TASK_TYPE_MEMTABLE, + TASK_TYPE_NORMAL, + NUM_TASK_TYPES, + TASK_TYPE_FIRST = TASK_TYPE_MEMTABLE +} task_type; + +typedef struct task_system_config { + bool32 use_stats; + uint64 num_background_threads[NUM_TASK_TYPES]; +} task_system_config; + +platform_status +task_system_config_init(task_system_config *task_cfg, + bool32 use_stats, + const uint64 num_background_threads[NUM_TASK_TYPES]); + typedef struct task_system task_system; -typedef void (*task_hook)(task_system *arg); -typedef void (*task_fn)(void *arg, void *scratch); +typedef void (*task_fn)(void *arg); typedef struct task { struct task *next; @@ -63,42 +91,12 @@ typedef struct task_group { task_stats stats[MAX_THREADS]; } task_group; -/* - * We maintain separate task groups for the memtable because memtable - * jobs are much shorter than other jobs and are latency critical. By - * separating them out, we can devote a small number of threads to - * deal with these small, latency critical tasks. - */ -typedef enum task_type { - TASK_TYPE_INVALID = 0, - TASK_TYPE_MEMTABLE, - TASK_TYPE_NORMAL, - NUM_TASK_TYPES, - TASK_TYPE_FIRST = TASK_TYPE_MEMTABLE -} task_type; - -typedef struct task_system_config { - bool32 use_stats; - uint64 num_background_threads[NUM_TASK_TYPES]; - uint64 scratch_size; -} task_system_config; - -platform_status -task_system_config_init(task_system_config *task_cfg, - bool32 use_stats, - const uint64 num_background_threads[NUM_TASK_TYPES], - uint64 scratch_size); - - -#define TASK_MAX_HOOKS (4) - /* * ---------------------------------------------------------------------- * Splinter specific state that gets created during initialization in * splinter_system_init(). Contains global state for splinter such as the - * init thread, init thread's scratch memory, thread_id counter and an array - * of all the threads, which acts like a map that is accessed by thread ID - * to get the thread pointer. + * init thread, thread_id counter and an array of all the threads, which + * acts like a map that is accessed by thread ID to get the thread pointer. * * This structure is passed around like an opaque structure to all the * entities that need to access it. Some of them are task creation and @@ -107,71 +105,16 @@ task_system_config_init(task_system_config *task_cfg, */ struct task_system { const task_system_config *cfg; - // array of scratch space pointers for this system. - // IO handle (currently one splinter system has just one) - platform_io_handle *ioh; - platform_heap_id heap_id; - /* - * bitmask used for allocating thread id's. - * If a bit is set to 0, it means we have an in use thread id for that - * particular position, 1 means it is unset and that thread id is available - * for use. - */ - uint64 tid_bitmask_lock; - uint64 tid_bitmask[(MAX_THREADS + 63) / 64]; - // max thread id so far. - threadid max_tid; - void *thread_scratch[MAX_THREADS]; + platform_heap_id heap_id; // task groups task_group group[NUM_TASK_TYPES]; - - int hook_init_done; - int num_hooks; - task_hook hooks[TASK_MAX_HOOKS]; }; -platform_status -task_thread_create(const char *name, - platform_thread_worker func, - void *arg, - size_t scratch_size, - task_system *ts, - platform_heap_id hid, - platform_thread *thread); - -/* - * Thread registration and deregistration. These functions are for - * threads created outside of the task system. Threads created with - * task_thread_create are automatically registered and unregistered. - */ - -// Register the calling thread, allocating scratch space for it -#define task_register_this_thread(ts, scratch_size) \ - task_register_thread((ts), (scratch_size), __FILE__, __LINE__, __func__) - -platform_status -task_register_thread(task_system *ts, - uint64 scratch_size, - const char *file, - const int lineno, - const char *func); - -// Unregister the calling thread and free its scratch space -#define task_deregister_this_thread(ts) \ - task_deregister_thread((ts), __FILE__, __LINE__, __func__) - -void -task_deregister_thread(task_system *ts, - const char *file, - const int lineno, - const char *func); - /* * Create a task system and register the calling thread. */ platform_status task_system_create(platform_heap_id hid, - platform_io_handle *ioh, task_system **system, const task_system_config *cfg); @@ -192,10 +135,6 @@ task_system_create(platform_heap_id hid, void task_system_destroy(platform_heap_id hid, task_system **ts); - -void * -task_system_get_thread_scratch(task_system *ts, threadid tid); - platform_status task_enqueue(task_system *ts, task_type type, @@ -265,8 +204,5 @@ task_perform_until_quiescent(task_system *ts); *Functions for tests and debugging. */ -threadid -task_get_max_tid(task_system *ts); - void task_print_stats(task_system *ts); diff --git a/src/trunk.c b/src/trunk.c index 36bfc584f..02a1e5a9b 100644 --- a/src/trunk.c +++ b/src/trunk.c @@ -1,4 +1,4 @@ -// Copyright 2018-2021 VMware, Inc. +// Copyright 2018-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* @@ -9,7 +9,6 @@ #include "trunk.h" #include "platform.h" -#include "platform_types.h" #include "data_internal.h" #include "util.h" #include "btree.h" @@ -2354,13 +2353,13 @@ trunk_branch_merger_deinit(trunk_branch_merger *merger) static void trunk_read_begin(trunk_context *context) { - platform_batch_rwlock_get(&context->root_lock, 0); + batch_rwlock_get(&context->root_lock, 0); } static void trunk_read_end(trunk_context *context) { - platform_batch_rwlock_unget(&context->root_lock, 0); + batch_rwlock_unget(&context->root_lock, 0); } platform_status @@ -2391,18 +2390,18 @@ trunk_ondisk_node_handle_addr(const trunk_ondisk_node_handle *handle) void trunk_modification_begin(trunk_context *context) { - platform_batch_rwlock_get(&context->root_lock, 0); - platform_batch_rwlock_claim_loop(&context->root_lock, 0); + batch_rwlock_get(&context->root_lock, 0); + batch_rwlock_claim_loop(&context->root_lock, 0); } static void trunk_set_root(trunk_context *context, trunk_ondisk_node_ref *new_root_ref) { trunk_ondisk_node_ref *old_root_ref; - platform_batch_rwlock_lock(&context->root_lock, 0); + batch_rwlock_lock(&context->root_lock, 0); old_root_ref = context->root; context->root = new_root_ref; - platform_batch_rwlock_unlock(&context->root_lock, 0); + batch_rwlock_unlock(&context->root_lock, 0); if (old_root_ref != NULL) { trunk_ondisk_node_ref_destroy(old_root_ref, context, context->hid); } @@ -2433,8 +2432,8 @@ perform_pending_gcs(trunk_context *context) void trunk_modification_end(trunk_context *context) { - platform_batch_rwlock_unclaim(&context->root_lock, 0); - platform_batch_rwlock_unget(&context->root_lock, 0); + batch_rwlock_unclaim(&context->root_lock, 0); + batch_rwlock_unget(&context->root_lock, 0); perform_pending_gcs(context); } @@ -2928,7 +2927,7 @@ trunk_pivot_state_map_create_entry(trunk_context *context, state->maplet = pivot_bundle->maplet; routing_filter_inc_ref(context->cc, &state->maplet); state->num_branches = bundle_num_branches(pivot_bundle); - platform_spinlock_init(&state->compactions_lock, NULL, context->hid); + platform_spinlock_init(&state->compactions_lock); state->next = map->buckets[*lock]; map->buckets[*lock] = state; @@ -3152,7 +3151,7 @@ static platform_status enqueue_maplet_compaction(trunk_pivot_state *args); static void -maplet_compaction_task(void *arg, void *scratch) +maplet_compaction_task(void *arg) { platform_status rc = STATUS_OK; trunk_pivot_state *state = (trunk_pivot_state *)arg; @@ -3373,7 +3372,7 @@ compute_tuple_bound(trunk_context *context, static void -bundle_compaction_task(void *arg, void *scratch) +bundle_compaction_task(void *arg) { platform_status rc; trunk_pivot_state *state = (trunk_pivot_state *)arg; @@ -4945,12 +4944,12 @@ trunk_incorporate_prepare(trunk_context *context, uint64 branch_addr) void trunk_incorporate_commit(trunk_context *context) { - platform_batch_rwlock_lock(&context->root_lock, 0); + batch_rwlock_lock(&context->root_lock, 0); platform_assert(context->pre_incorporation_root == NULL); context->pre_incorporation_root = context->root; context->root = context->post_incorporation_root; context->post_incorporation_root = NULL; - platform_batch_rwlock_unlock(&context->root_lock, 0); + batch_rwlock_unlock(&context->root_lock, 0); } void @@ -5793,7 +5792,7 @@ trunk_context_init(trunk_context *context, } trunk_pivot_state_map_init(&context->pivot_states); - platform_batch_rwlock_init(&context->root_lock); + batch_rwlock_init(&context->root_lock); return STATUS_OK; } @@ -5852,7 +5851,7 @@ trunk_context_deinit(trunk_context *context) perform_pending_gcs(context); platform_assert(context->pending_gcs == NULL); trunk_pivot_state_map_deinit(&context->pivot_states); - platform_batch_rwlock_deinit(&context->root_lock); + batch_rwlock_deinit(&context->root_lock); } diff --git a/src/trunk.h b/src/trunk.h index a6d536359..f7e125755 100644 --- a/src/trunk.h +++ b/src/trunk.h @@ -1,4 +1,4 @@ -// Copyright 2023 VMware, Inc. +// Copyright 2023-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* @@ -7,7 +7,6 @@ * This file contains the interface of the SplinterDB trunk. */ -#include "platform.h" #include "vector.h" #include "cache.h" #include "allocator.h" @@ -17,6 +16,7 @@ #include "iterator.h" #include "merge.h" #include "data_internal.h" +#include "batch_rwlock.h" typedef struct trunk_config { const data_config *data_cfg; @@ -163,7 +163,7 @@ typedef struct trunk_context { task_system *ts; trunk_stats *stats; trunk_pivot_state_map pivot_states; - platform_batch_rwlock root_lock; + batch_rwlock root_lock; trunk_ondisk_node_ref *root; trunk_ondisk_node_ref *post_incorporation_root; trunk_ondisk_node_ref *pre_incorporation_root; diff --git a/src/util.c b/src/util.c index badad579c..69bf4c565 100644 --- a/src/util.c +++ b/src/util.c @@ -1,9 +1,15 @@ -// Copyright 2018-2021 VMware, Inc. +// Copyright 2018-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 -#include "platform.h" - #include "util.h" +#include "splinterdb/platform_linux/public_platform.h" +#include "platform_status.h" +#include "platform_log.h" +#include "platform_assert.h" +#include "platform_util.h" +#include "platform_typed_alloc.h" +#include "platform_units.h" +#include "platform_string.h" #include "poison.h" diff --git a/src/util.h b/src/util.h index be333b73f..4fb303443 100644 --- a/src/util.h +++ b/src/util.h @@ -1,10 +1,10 @@ -// Copyright 2018-2021 VMware, Inc. +// Copyright 2018-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 #pragma once -#include "platform.h" #include "splinterdb/public_util.h" +#include "platform_heap.h" #define bitsizeof(x) (8 * sizeof(x)) @@ -51,32 +51,6 @@ int64abs(int64 j) return (j >= 0) ? (uint64)j : ((uint64) - (j + 1)) + 1; } -typedef struct fraction { - uint64 numerator; - uint64 denominator; -} fraction; - -static inline fraction -init_fraction(uint64 numerator, uint64 denominator) -{ - return (fraction){ - .numerator = numerator, - .denominator = denominator, - }; -} - -#define zero_fraction \ - ((fraction){ \ - .numerator = 0, \ - .denominator = 1, \ - }) - -static inline fraction -fraction_init_or_zero(uint64 num, uint64 den) -{ - return den ? init_fraction(num, den) : zero_fraction; -} - static inline slice slice_copy_contents(void *dst, const slice src) { @@ -397,42 +371,6 @@ debug_hex_dump_slice(platform_log_handle *, uint64 grouping, slice data); : ((intval) < 1000) ? "3d" \ : "4d") -// Length of output buffer to snprintf()-into size as string w/ unit specifier -#define SIZE_TO_STR_LEN 20 - -// Format a size value with unit-specifiers, in an output buffer. -char * -size_to_str(char *outbuf, size_t outbuflen, size_t size); - -char * -size_to_fmtstr(char *outbuf, size_t outbuflen, const char *fmtstr, size_t size); - -/* - * Convenience caller macros to convert 'sz' bytes to return a string, - * formatting the input size as human-readable value with unit-specifiers. - */ -// char *size_str(size_t sz) -#define size_str(sz) \ - (({ \ - struct { \ - char buffer[SIZE_TO_STR_LEN]; \ - } onstack_chartmp; \ - size_to_str( \ - onstack_chartmp.buffer, sizeof(onstack_chartmp.buffer), sz); \ - onstack_chartmp; \ - }).buffer) - -// char *size_fmtstr(const char *fmtstr, size_t sz) -#define size_fmtstr(fmtstr, sz) \ - (({ \ - struct { \ - char buffer[SIZE_TO_STR_LEN]; \ - } onstack_chartmp; \ - size_to_fmtstr( \ - onstack_chartmp.buffer, sizeof(onstack_chartmp.buffer), fmtstr, sz); \ - onstack_chartmp; \ - }).buffer) - /************************************ * Helpers for statistics ************************************/ diff --git a/src/vector.h b/src/vector.h index 5ac92a61c..77c87fecd 100644 --- a/src/vector.h +++ b/src/vector.h @@ -1,3 +1,6 @@ +// Copyright 2018-2026 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + /* * Type-safe vectors. Implementation is entirely macros. * @@ -168,7 +171,7 @@ __vector_replace(writable_buffer *dst, uint64 __start = (start); \ vector_elt_ptr_type(src) __srcdata = vector_data(src); \ writable_buffer_append(&(dst)->wb, \ - ((end)-__start) * vector_elt_size(src), \ + ((end) - __start) * vector_elt_size(src), \ __srcdata + __start); \ }) @@ -186,7 +189,7 @@ __vector_replace(writable_buffer *dst, }) #define vector_ensure_capacity(v, capacity) \ - (writable_buffer_ensure_space(&(v)->wb, (capacity)*vector_elt_size(v))) + (writable_buffer_ensure_space(&(v)->wb, (capacity) * vector_elt_size(v))) #define vector_copy(v, src) \ ({ \ diff --git a/tests/config.c b/tests/config.c index d7a83c2d6..3e3101398 100644 --- a/tests/config.c +++ b/tests/config.c @@ -1,7 +1,9 @@ -// Copyright 2018-2021 VMware, Inc. +// Copyright 2018-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 #include "config.h" +#include "platform_units.h" +#include "platform_buffer.h" #include "util.h" /* diff --git a/tests/config.h b/tests/config.h index 69f9703cd..cd513f36b 100644 --- a/tests/config.h +++ b/tests/config.h @@ -1,4 +1,4 @@ -// Copyright 2018-2021 VMware, Inc. +// Copyright 2018-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* @@ -20,9 +20,9 @@ extern const char *BUILD_VERSION; * master_config used to run tests. * -------------------------------------------------------------------------- */ -#define TEST_CONFIG_DEFAULT_PAGE_SIZE LAIO_DEFAULT_PAGE_SIZE // bytes +#define TEST_CONFIG_DEFAULT_PAGE_SIZE IO_DEFAULT_PAGE_SIZE // bytes -#define TEST_CONFIG_DEFAULT_PAGES_PER_EXTENT LAIO_DEFAULT_PAGES_PER_EXTENT +#define TEST_CONFIG_DEFAULT_PAGES_PER_EXTENT IO_DEFAULT_PAGES_PER_EXTENT _Static_assert(TEST_CONFIG_DEFAULT_PAGES_PER_EXTENT <= MAX_PAGES_PER_EXTENT, "Invalid TEST_CONFIG_DEFAULT_PAGES_PER_EXTENT value"); diff --git a/tests/functional/avlTree.c b/tests/functional/avlTree.c index 79b2b7a9e..dadc48f57 100644 --- a/tests/functional/avlTree.c +++ b/tests/functional/avlTree.c @@ -1,4 +1,4 @@ -// Copyright 2018-2021 VMware, Inc. +// Copyright 2018-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* diff --git a/tests/functional/avlTree.h b/tests/functional/avlTree.h index a71c3163c..3e4a6c208 100644 --- a/tests/functional/avlTree.h +++ b/tests/functional/avlTree.h @@ -1,4 +1,4 @@ -// Copyright 2018-2021 VMware, Inc. +// Copyright 2018-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* diff --git a/tests/functional/btree_test.c b/tests/functional/btree_test.c index 2e21c02f4..1f1b33f27 100644 --- a/tests/functional/btree_test.c +++ b/tests/functional/btree_test.c @@ -1,4 +1,4 @@ -// Copyright 2018-2021 VMware, Inc. +// Copyright 2018-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* @@ -6,13 +6,13 @@ * * This file contains the test interfaces for Alex's B-tree. */ -#include "platform.h" #include "splinterdb/data.h" +#include "platform_time.h" #include "btree.h" #include "merge.h" #include "test.h" -#include "io.h" +#include "platform_io.h" #include "allocator.h" #include "rc_allocator.h" #include "cache.h" @@ -245,20 +245,18 @@ test_btree_perf(cache *cc, } for (uint64 thread_no = 0; thread_no < num_threads; thread_no++) { - ret = task_thread_create("insert thread", - test_btree_insert_thread, - ¶ms[thread_no], - 0, - ts, - hid, - ¶ms[thread_no].thread); + ret = platform_thread_create(¶ms[thread_no].thread, + FALSE, + test_btree_insert_thread, + ¶ms[thread_no], + hid); if (!SUCCESS(ret)) { return ret; } } for (uint64 thread_no = 0; thread_no < num_threads; thread_no++) { - platform_thread_join(params[thread_no].thread); + platform_thread_join(¶ms[thread_no].thread); } for (uint64 thread_no = 0; thread_no < num_threads; thread_no++) { @@ -1527,6 +1525,8 @@ btree_test(int argc, char *argv[]) task_system *ts = NULL; test_message_generator gen; + platform_register_thread(); + if (argc > 1 && strncmp(argv[1], "--perf", sizeof("--perf")) == 0) { run_perf_test = TRUE; config_argc = argc - 2; @@ -1600,31 +1600,28 @@ btree_test(int argc, char *argv[]) } } - platform_io_handle *io = TYPED_MALLOC(hid, io); - platform_assert(io != NULL); - rc = io_handle_init(io, &system_cfg.io_cfg, hid); - if (!SUCCESS(rc)) { - goto free_iohandle; + io_handle *io = io_handle_create(&system_cfg.io_cfg, hid); + if (io == NULL) { + platform_error_log("Failed to create IO handle\n"); + rc = STATUS_NO_MEMORY; + goto cleanup; } - rc = test_init_task_system(hid, io, &ts, &system_cfg.task_cfg); + rc = test_init_task_system(hid, &ts, &system_cfg.task_cfg); if (!SUCCESS(rc)) { platform_error_log("Failed to init splinter state: %s\n", platform_status_to_string(rc)); - goto deinit_iohandle; + goto destroy_iohandle; } rc_allocator al; - rc_allocator_init(&al, - &system_cfg.allocator_cfg, - (io_handle *)io, - hid, - platform_get_module_id()); + rc_allocator_init( + &al, &system_cfg.allocator_cfg, io, hid, platform_get_module_id()); clockcache *cc = TYPED_MALLOC(hid, cc); rc = clockcache_init(cc, &system_cfg.cache_cfg, - (io_handle *)io, + io, (allocator *)&al, "test", hid, @@ -1672,12 +1669,10 @@ btree_test(int argc, char *argv[]) rc_allocator_deinit(&al); test_deinit_task_system(hid, &ts); rc = STATUS_OK; -deinit_iohandle: - io_handle_deinit(io); -free_iohandle: - platform_free(hid, io); +destroy_iohandle: + io_handle_destroy(io); cleanup: platform_heap_destroy(&hid); - + platform_deregister_thread(); return SUCCESS(rc) ? 0 : -1; } diff --git a/tests/functional/cache_test.c b/tests/functional/cache_test.c index 66a4605c8..5a2bd126f 100644 --- a/tests/functional/cache_test.c +++ b/tests/functional/cache_test.c @@ -1,4 +1,4 @@ -// Copyright 2018-2021 VMware, Inc. +// Copyright 2018-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* @@ -7,16 +7,12 @@ * This file contains the tests for clockcache. */ -#include "platform.h" - #include "test.h" #include "allocator.h" #include "rc_allocator.h" #include "cache.h" #include "clockcache.h" -#include "splinterdb/data.h" #include "task.h" -#include "util.h" #include "random.h" #include "test_common.h" #include "poison.h" @@ -852,21 +848,11 @@ test_cache_async(cache *cc, */ params[i].mt_reader = total_threads > 1 ? TRUE : FALSE; if (is_reader) { - rc = task_thread_create("cache_reader", - test_reader_thread, - ¶ms[i], - 0, - ts, - hid, - ¶ms[i].thread); + rc = platform_thread_create( + ¶ms[i].thread, FALSE, test_reader_thread, ¶ms[i], hid); } else { - rc = task_thread_create("cache_writer", - test_writer_thread, - ¶ms[i], - 0, - ts, - hid, - ¶ms[i].thread); + rc = platform_thread_create( + ¶ms[i].thread, FALSE, test_writer_thread, ¶ms[i], hid); } if (!SUCCESS(rc)) { total_threads = i; @@ -875,7 +861,7 @@ test_cache_async(cache *cc, } // Wait for test threads for (i = 0; i < total_threads; i++) { - platform_thread_join(params[i].thread); + platform_thread_join(¶ms[i].thread); } for (i = 0; i < pages_to_allocate; i++) { @@ -924,6 +910,8 @@ cache_test(int argc, char *argv[]) uint64 seed; test_message_generator gen; + platform_register_thread(); + if (argc > 1) { if (strncmp(argv[1], "--perf", sizeof("--perf")) == 0) { benchmark = TRUE; @@ -981,31 +969,28 @@ cache_test(int argc, char *argv[]) goto cleanup; } - platform_io_handle *io = TYPED_MALLOC(hid, io); - platform_assert(io != NULL); - rc = io_handle_init(io, &system_cfg.io_cfg, hid); - if (!SUCCESS(rc)) { - goto free_iohandle; + io_handle *io = io_handle_create(&system_cfg.io_cfg, hid); + if (io == NULL) { + platform_error_log("Failed to create IO handle\n"); + rc = STATUS_NO_MEMORY; + goto cleanup; } - rc = test_init_task_system(hid, io, &ts, &system_cfg.task_cfg); + rc = test_init_task_system(hid, &ts, &system_cfg.task_cfg); if (!SUCCESS(rc)) { platform_error_log("Failed to init splinter state: %s\n", platform_status_to_string(rc)); - goto deinit_iohandle; + goto destroy_iohandle; } rc_allocator al; - rc_allocator_init(&al, - &system_cfg.allocator_cfg, - (io_handle *)io, - hid, - platform_get_module_id()); + rc_allocator_init( + &al, &system_cfg.allocator_cfg, io, hid, platform_get_module_id()); clockcache *cc = TYPED_MALLOC(hid, cc); rc = clockcache_init(cc, &system_cfg.cache_cfg, - (io_handle *)io, + io, (allocator *)&al, "test", hid, @@ -1083,13 +1068,11 @@ cache_test(int argc, char *argv[]) rc_allocator_deinit(&al); test_deinit_task_system(hid, &ts); rc = STATUS_OK; -deinit_iohandle: - io_handle_deinit(io); -free_iohandle: - platform_free(hid, io); +destroy_iohandle: + io_handle_destroy(io); cleanup: platform_free(hid, splinter_cfg); platform_heap_destroy(&hid); - + platform_deregister_thread(); return SUCCESS(rc) ? 0 : -1; } diff --git a/tests/functional/driver_test.c b/tests/functional/driver_test.c index 7b1666f16..8f6ffabe9 100644 --- a/tests/functional/driver_test.c +++ b/tests/functional/driver_test.c @@ -1,4 +1,4 @@ -// Copyright 2018-2021 VMware, Inc. +// Copyright 2018-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 #include "platform.h" diff --git a/tests/functional/filter_test.c b/tests/functional/filter_test.c index b64e9cce4..48b672fc9 100644 --- a/tests/functional/filter_test.c +++ b/tests/functional/filter_test.c @@ -1,4 +1,4 @@ -// Copyright 2018-2021 VMware, Inc. +// Copyright 2018-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* @@ -6,18 +6,17 @@ * * This file contains the test interfaces for Alex's filter */ -#include "platform.h" - #include "splinterdb/data.h" #include "test.h" #include "routing_filter.h" #include "allocator.h" #include "rc_allocator.h" -#include "shard_log.h" #include "cache.h" #include "clockcache.h" #include "util.h" - +#include "platform_time.h" +#include "platform_typed_alloc.h" +#include "platform_assert.h" #include "poison.h" static platform_status @@ -291,6 +290,8 @@ filter_test(int argc, char *argv[]) uint64 seed; test_message_generator gen; + platform_register_thread(); + if (argc > 1 && strncmp(argv[1], "--perf", sizeof("--perf")) == 0) { run_perf_test = TRUE; config_argc = argc - 2; @@ -330,29 +331,27 @@ filter_test(int argc, char *argv[]) goto cleanup; } - platform_io_handle *io = TYPED_MALLOC(hid, io); - platform_assert(io != NULL); - rc = io_handle_init(io, &system_cfg.io_cfg, hid); - if (!SUCCESS(rc)) { - goto free_iohandle; + io_handle *io = io_handle_create(&system_cfg.io_cfg, hid); + if (io == NULL) { + platform_error_log("Failed to create IO handle\n"); + rc = STATUS_NO_MEMORY; + r = -1; + goto cleanup; } task_system *ts = NULL; - rc = task_system_create(hid, io, &ts, &system_cfg.task_cfg); + rc = task_system_create(hid, &ts, &system_cfg.task_cfg); platform_assert_status_ok(rc); - rc = rc_allocator_init(&al, - &system_cfg.allocator_cfg, - (io_handle *)io, - hid, - platform_get_module_id()); + rc = rc_allocator_init( + &al, &system_cfg.allocator_cfg, io, hid, platform_get_module_id()); platform_assert_status_ok(rc); cc = TYPED_MALLOC(hid, cc); platform_assert(cc); rc = clockcache_init(cc, &system_cfg.cache_cfg, - (io_handle *)io, + io, (allocator *)&al, "test", hid, @@ -401,12 +400,10 @@ filter_test(int argc, char *argv[]) platform_free(hid, cc); rc_allocator_deinit(&al); task_system_destroy(hid, &ts); - io_handle_deinit(io); -free_iohandle: - platform_free(hid, io); + io_handle_destroy(io); r = 0; cleanup: platform_heap_destroy(&hid); - + platform_deregister_thread(); return r; } diff --git a/tests/functional/io_apis_test.c b/tests/functional/io_apis_test.c index 3335ebfb9..c00669643 100644 --- a/tests/functional/io_apis_test.c +++ b/tests/functional/io_apis_test.c @@ -1,4 +1,4 @@ -// Copyright 2018-2022 VMware, Inc. +// Copyright 2018-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* @@ -34,10 +34,11 @@ */ #include -#include "platform.h" +#include "platform_units.h" +#include "platform_typed_alloc.h" +#include "platform_sleep.h" #include "config.h" -#include "io.h" -#include "core.h" // Needed for trunk_get_scratch_size() +#include "platform_io.h" #include "task.h" /* @@ -47,15 +48,15 @@ * Most fields are used by all functions receiving this arg. Some are for diags. */ typedef struct io_test_fn_args { - platform_heap_id hid; - io_config *io_cfgp; - platform_io_handle *io_hdlp; - task_system *tasks; - uint64 start_addr; - uint64 end_addr; - char stamp_char; - platform_thread thread; - const char *whoami; // 'Parent' or 'Child' + platform_heap_id hid; + io_config *io_cfgp; + io_handle *io_hdlp; + task_system *tasks; + uint64 start_addr; + uint64 end_addr; + char stamp_char; + platform_thread thread; + const char *whoami; // 'Parent' or 'Child' } io_test_fn_args; /* Whether to display verbose-progress from each thread's activity */ @@ -65,7 +66,7 @@ bool32 Verbose_progress = FALSE; * Global to track address of allocated IO-handle allocated in parent process. * Used for cross-checking in child's context after fork(). */ -platform_io_handle *Parent_io_handle = NULL; +io_handle *Parent_io_handle = NULL; /* * Different test cases in this test drive multiple threads each doing one @@ -97,20 +98,20 @@ typedef void (*test_io_thread_hdlr)(void *arg); // Function prototypes static platform_status -test_sync_writes(platform_heap_id hid, - io_config *io_cfgp, - platform_io_handle *io_hdlp, - uint64 start_addr, - uint64 end_addr, - char stamp_char); +test_sync_writes(platform_heap_id hid, + io_config *io_cfgp, + io_handle *io_hdlp, + uint64 start_addr, + uint64 end_addr, + char stamp_char); static platform_status -test_sync_reads(platform_heap_id hid, - io_config *io_cfgp, - platform_io_handle *io_hdlp, - uint64 start_addr, - uint64 end_addr, - char stamp_char); +test_sync_reads(platform_heap_id hid, + io_config *io_cfgp, + io_handle *io_hdlp, + uint64 start_addr, + uint64 end_addr, + char stamp_char); static platform_status test_sync_write_reads_by_threads(io_test_fn_args *io_test_param, @@ -118,12 +119,12 @@ test_sync_write_reads_by_threads(io_test_fn_args *io_test_param, const char *whoami); static platform_status -test_async_reads(platform_heap_id hid, - io_config *io_cfgp, - platform_io_handle *io_hdlp, - uint64 start_addr, - char stamp_char, - const char *whoami); +test_async_reads(platform_heap_id hid, + io_config *io_cfgp, + io_handle *io_hdlp, + uint64 start_addr, + char stamp_char, + const char *whoami); static platform_status test_async_reads_by_threads(io_test_fn_args *io_test_param, @@ -174,6 +175,7 @@ npages_per_thread(io_test_fn_args *io_test_param, int nthreads) int splinter_io_apis_test(int argc, char *argv[]) { + platform_register_thread(); uint64 heap_capacity = (HEAP_SIZE_MB * MiB); // small heap is sufficient. // Move past the 1st arg which will be the driving tag, 'io_apis_test'. @@ -222,7 +224,7 @@ splinter_io_apis_test(int argc, char *argv[]) master_cfg.io_async_queue_depth, "splinterdb_io_apis_test_db"); - int pid = platform_getpid(); + int pid = platform_get_os_pid(); platform_default_log("Parent OS-pid=%d, Exercise IO sub-system test on" " device '%s'" ", page_size=%lu, extent_size=%lu" @@ -235,21 +237,14 @@ splinter_io_apis_test(int argc, char *argv[]) // For this test, we allocate this structure. In a running Splinter // instance, this struct is nested inside the splinterdb{} handle. - platform_io_handle *io_hdl = TYPED_ZALLOC(hid, io_hdl); - if (!io_hdl) { + io_handle *io_hdl = io_handle_create(&io_cfg, hid); + if (io_hdl == NULL) { + platform_error_log("Failed to create IO handle\n"); + rc = STATUS_NO_MEMORY; goto heap_destroy; } Parent_io_handle = io_hdl; - // Initialize the handle to the IO sub-system. A device with a small initial - // size gets created here. - rc = io_handle_init(io_hdl, &io_cfg, hid); - if (!SUCCESS(rc)) { - platform_error_log("Failed to initialize IO handle: %s\n", - platform_status_to_string(rc)); - goto io_free; - } - uint64 disk_size_MB = DEVICE_SIZE_MB; uint64 disk_size = (disk_size_MB * MiB); // bytes @@ -268,12 +263,12 @@ splinter_io_apis_test(int argc, char *argv[]) */ uint64 num_bg_threads[NUM_TASK_TYPES] = {0}; task_system_config task_cfg; - rc = task_system_config_init( - &task_cfg, TRUE /* use stats */, num_bg_threads, core_get_scratch_size()); + rc = + task_system_config_init(&task_cfg, TRUE /* use stats */, num_bg_threads); platform_assert(SUCCESS(rc)); task_system *tasks = NULL; - rc = task_system_create(hid, io_hdl, &tasks, &task_cfg); + rc = task_system_create(hid, &tasks, &task_cfg); platform_assert(SUCCESS(rc)); threadid main_thread_idx = platform_get_tid(); @@ -329,7 +324,7 @@ splinter_io_apis_test(int argc, char *argv[]) "Child execution wait() completed." " Resuming parent ...\n", platform_get_tid(), - platform_getpid()); + platform_get_os_pid()); } } } @@ -337,6 +332,9 @@ splinter_io_apis_test(int argc, char *argv[]) // Exercise R/W tests: Run this block if we are in the main thread // (i.e. didn't fork) or if we are a child process created after fork()'ing. if (!master_cfg.fork_child || (pid == 0)) { + if (pid == 0) { + platform_register_thread(); + } threadid this_thread_idx = platform_get_tid(); @@ -349,14 +347,13 @@ splinter_io_apis_test(int argc, char *argv[]) ", ThreadID=%lu (%s)" ", before thread registration" ", Parent io_handle=%p, io_hdl=%p\n", - platform_getpid(), + platform_get_os_pid(), this_thread_idx, whoami, Parent_io_handle, io_hdl); } - task_register_this_thread(tasks, core_get_scratch_size()); this_thread_idx = platform_get_tid(); // Reset the handles / variables that have changed in the child @@ -368,7 +365,7 @@ splinter_io_apis_test(int argc, char *argv[]) platform_default_log("After fork()'ing: %s OS-pid=%d" ", ThreadID=%lu", whoami, - platform_getpid(), + platform_get_os_pid(), this_thread_idx); if (Verbose_progress) { @@ -390,22 +387,16 @@ splinter_io_apis_test(int argc, char *argv[]) platform_assert_status_ok(rc); test_async_reads_by_threads(&io_test_fn_arg, NUM_THREADS, whoami); - - // The forked child process which uses Splinter masquerading as a - // "thread" needs to relinquish its resources before exiting. - task_deregister_this_thread(tasks); } +io_free: // Only the parent process should dismantle stuff - if (pid > 0) { + if (pid != 0) { task_system_destroy(hid, &tasks); - io_handle_deinit(io_hdl); + io_handle_destroy(io_hdl); } -io_free: - if (pid > 0) { - platform_free(hid, io_hdl); - } + heap_destroy: if (pid > 0) { platform_heap_destroy(&hid); @@ -413,9 +404,10 @@ splinter_io_apis_test(int argc, char *argv[]) platform_default_log("%s: OS-pid=%d, Thread-ID=%lu" " execution completed.\n", whoami, - platform_getpid(), + platform_get_os_pid(), platform_get_tid()); } + platform_deregister_thread(); return (SUCCESS(rc) ? 0 : -1); } @@ -437,12 +429,12 @@ splinter_io_apis_test(int argc, char *argv[]) * ----------------------------------------------------------------------------- */ static platform_status -test_sync_writes(platform_heap_id hid, - io_config *io_cfgp, - platform_io_handle *io_hdlp, - uint64 start_addr, - uint64 end_addr, - char stamp_char) +test_sync_writes(platform_heap_id hid, + io_config *io_cfgp, + io_handle *io_hdlp, + uint64 start_addr, + uint64 end_addr, + char stamp_char) { platform_thread this_thread = platform_get_tid(); platform_status rc = STATUS_NO_MEMORY; @@ -479,7 +471,7 @@ test_sync_writes(platform_heap_id hid, " %s(): OS-pid=%d, Thread %lu performed %lu %dK page write IOs " "from start addr=%lu through end addr=%lu\n", __func__, - platform_getpid(), + platform_get_os_pid(), this_thread, num_IOs, (int)(page_size / KiB), @@ -529,12 +521,12 @@ test_sync_writes_worker(void *arg) * ----------------------------------------------------------------------------- */ static platform_status -test_sync_reads(platform_heap_id hid, - io_config *io_cfgp, - platform_io_handle *io_hdlp, - uint64 start_addr, - uint64 end_addr, - char stamp_char) +test_sync_reads(platform_heap_id hid, + io_config *io_cfgp, + io_handle *io_hdlp, + uint64 start_addr, + uint64 end_addr, + char stamp_char) { platform_thread this_thread = platform_get_tid(); @@ -578,7 +570,7 @@ test_sync_reads(platform_heap_id hid, " %s(): OS-pid=%d, Thread %lu performed %lu %dK page read IOs " "from start addr=%lu through end addr=%lu\n", __func__, - platform_getpid(), + platform_get_os_pid(), this_thread, num_IOs, (int)(page_size / KiB), @@ -640,7 +632,7 @@ test_sync_write_reads_by_threads(io_test_fn_args *io_test_param, "\n%s(): %s process, OS-pid=%d, creates %d threads, %lu pages/thread \n", __func__, whoami, - platform_getpid(), + platform_get_os_pid(), nthreads, npages); @@ -656,7 +648,7 @@ test_sync_write_reads_by_threads(io_test_fn_args *io_test_param, } for (int i = 0; i < nthreads; i++) { - platform_thread_join(thread_params[i].thread); + platform_thread_join(&thread_params[i].thread); } platform_default_log( @@ -674,7 +666,7 @@ test_sync_write_reads_by_threads(io_test_fn_args *io_test_param, } for (int i = 0; i < nthreads; i++) { - platform_thread_join(thread_params[i].thread); + platform_thread_join(&thread_params[i].thread); } platform_default_log( @@ -814,12 +806,12 @@ read_async_callback(void *arg) } static platform_status -test_async_reads(platform_heap_id hid, - io_config *io_cfgp, - platform_io_handle *io_hdlp, - uint64 start_addr, - char stamp_char, - const char *whoami) +test_async_reads(platform_heap_id hid, + io_config *io_cfgp, + io_handle *io_hdlp, + uint64 start_addr, + char stamp_char, + const char *whoami) { platform_thread this_thread = platform_get_tid(); platform_status rc = STATUS_OK; @@ -933,7 +925,7 @@ test_async_reads_by_threads(io_test_fn_args *io_test_param, } for (int i = 0; i < nthreads; i++) { - platform_thread_join(thread_params[i].thread); + platform_thread_join(&thread_params[i].thread); } platform_default_log( @@ -976,13 +968,8 @@ do_n_thread_creates(const char *thread_type, { platform_status ret; for (uint64 i = 0; i < num_threads; i++) { - ret = task_thread_create(thread_type, - thread_hdlr, - ¶ms[i], - core_get_scratch_size(), - params[i].tasks, - params[i].hid, - ¶ms[i].thread); + ret = platform_thread_create( + ¶ms[i].thread, FALSE, thread_hdlr, ¶ms[i], params[i].hid); if (!SUCCESS(ret)) { return ret; } diff --git a/tests/functional/log_test.c b/tests/functional/log_test.c index d25902503..d2a8e8646 100644 --- a/tests/functional/log_test.c +++ b/tests/functional/log_test.c @@ -1,4 +1,4 @@ -// Copyright 2018-2021 VMware, Inc. +// Copyright 2018-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* @@ -6,11 +6,10 @@ * * This file contains tests for Alex's log */ -#include "platform.h" - +#include "platform_time.h" #include "log.h" #include "shard_log.h" -#include "io.h" +#include "platform_io.h" #include "allocator.h" #include "rc_allocator.h" #include "cache.h" @@ -181,23 +180,18 @@ test_log_perf(cache *cc, start_time = platform_get_timestamp(); for (uint64 i = 0; i < num_threads; i++) { - ret = task_thread_create("log_thread", - test_log_thread, - ¶ms[i], - 0, - ts, - hid, - ¶ms[i].thread); + ret = platform_thread_create( + ¶ms[i].thread, FALSE, test_log_thread, ¶ms[i], hid); if (!SUCCESS(ret)) { // Wait for existing threads to quit for (uint64 j = 0; j < i; j++) { - platform_thread_join(params[i].thread); + platform_thread_join(¶ms[j].thread); } goto cleanup; } } for (uint64 i = 0; i < num_threads; i++) { - platform_thread_join(params[i].thread); + platform_thread_join(¶ms[i].thread); } platform_default_log("log insertion rate: %luM insertions/second\n", @@ -240,6 +234,8 @@ log_test(int argc, char *argv[]) task_system *ts = NULL; test_message_generator gen; + platform_register_thread(); + if (argc > 1 && strncmp(argv[1], "--perf", sizeof("--perf")) == 0) { run_perf_test = TRUE; run_crash_test = FALSE; @@ -289,34 +285,30 @@ log_test(int argc, char *argv[]) goto cleanup; } - platform_io_handle *io = TYPED_MALLOC(hid, io); - platform_assert(io != NULL); - status = io_handle_init(io, &system_cfg.io_cfg, hid); - if (!SUCCESS(status)) { + io_handle *io = io_handle_create(&system_cfg.io_cfg, hid); + if (io == NULL) { + platform_error_log("Failed to create IO handle\n"); rc = -1; - goto free_iohandle; + goto cleanup; } - status = test_init_task_system(hid, io, &ts, &system_cfg.task_cfg); + status = test_init_task_system(hid, &ts, &system_cfg.task_cfg); if (!SUCCESS(status)) { platform_error_log("Failed to init splinter state: %s\n", platform_status_to_string(status)); rc = -1; - goto deinit_iohandle; + goto destroy_iohandle; } - status = rc_allocator_init(&al, - &system_cfg.allocator_cfg, - (io_handle *)io, - hid, - platform_get_module_id()); + status = rc_allocator_init( + &al, &system_cfg.allocator_cfg, io, hid, platform_get_module_id()); platform_assert_status_ok(status); clockcache *cc = TYPED_MALLOC(hid, cc); platform_assert(cc != NULL); status = clockcache_init(cc, &system_cfg.cache_cfg, - (io_handle *)io, + io, (allocator *)&al, "test", hid, @@ -333,7 +325,7 @@ log_test(int argc, char *argv[]) } else if (run_crash_test) { rc = test_log_crash(cc, &system_cfg.cache_cfg, - (io_handle *)io, + io, (allocator *)&al, &system_cfg.log_cfg, log, @@ -346,7 +338,7 @@ log_test(int argc, char *argv[]) } else { rc = test_log_crash(cc, &system_cfg.cache_cfg, - (io_handle *)io, + io, (allocator *)&al, &system_cfg.log_cfg, log, @@ -363,13 +355,11 @@ log_test(int argc, char *argv[]) platform_free(hid, cc); rc_allocator_deinit(&al); test_deinit_task_system(hid, &ts); -deinit_iohandle: - io_handle_deinit(io); -free_iohandle: - platform_free(hid, io); +destroy_iohandle: + io_handle_destroy(io); cleanup: platform_free(hid, cfg); platform_heap_destroy(&hid); - + platform_deregister_thread(); return rc == 0 ? 0 : -1; } diff --git a/tests/functional/random.h b/tests/functional/random.h index 9d8e2f038..944015080 100644 --- a/tests/functional/random.h +++ b/tests/functional/random.h @@ -1,8 +1,7 @@ -// Copyright 2018-2021 VMware, Inc. +// Copyright 2018-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 -#ifndef _SPLINTER_RANDOM_H_ -#define _SPLINTER_RANDOM_H_ +#pragma once #include "platform.h" @@ -84,5 +83,3 @@ random_bytes(random_state *rs, char *v, size_t n) memmove(v, &remainder, n); } } - -#endif diff --git a/tests/functional/splinter_test.c b/tests/functional/splinter_test.c index d6f4c8105..c929bd7fe 100644 --- a/tests/functional/splinter_test.c +++ b/tests/functional/splinter_test.c @@ -1,4 +1,4 @@ -// Copyright 2018-2021 VMware, Inc. +// Copyright 2018-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* @@ -7,10 +7,7 @@ * This file contains the test interfaces for SplinterDB. */ -#include "platform.h" - #include "core.h" -#include "merge.h" #include "test.h" #include "allocator.h" #include "rc_allocator.h" @@ -900,13 +897,8 @@ do_n_thread_creates(const char *thread_type, { platform_status ret; for (uint64 i = 0; i < num_threads; i++) { - ret = task_thread_create(thread_type, - thread_hdlr, - ¶ms[i], - core_get_scratch_size(), - ts, - hid, - ¶ms[i].thread); + ret = platform_thread_create( + ¶ms[i].thread, FALSE, thread_hdlr, ¶ms[i], hid); if (!SUCCESS(ret)) { return ret; } @@ -1024,7 +1016,7 @@ splinter_perf_inserts(platform_heap_id hid, } for (uint64 i = 0; i < num_insert_threads; i++) { - platform_thread_join(params[i].thread); + platform_thread_join(¶ms[i].thread); } task_perform_until_quiescent(ts); @@ -1125,7 +1117,7 @@ splinter_perf_lookups(platform_heap_id hid, } for (uint64 i = 0; i < num_lookup_threads; i++) { - platform_thread_join(params[i].thread); + platform_thread_join(¶ms[i].thread); } uint64 total_time = platform_timestamp_elapsed(start_time); @@ -1270,7 +1262,7 @@ splinter_perf_range_lookups(platform_heap_id hid, } for (uint64 i = 0; i < num_range_threads; i++) { - platform_thread_join(params[i].thread); + platform_thread_join(¶ms[i].thread); } uint64 total_time = platform_timestamp_elapsed(start_time); @@ -1533,19 +1525,14 @@ test_splinter_periodic(system_config *cfg, for (uint64 i = 0; i < num_insert_threads; i++) { platform_status ret; - ret = task_thread_create("insert_thread", - test_trunk_insert_thread, - ¶ms[i], - core_get_scratch_size(), - ts, - hid, - ¶ms[i].thread); + ret = platform_thread_create( + ¶ms[i].thread, FALSE, test_trunk_insert_thread, ¶ms[i], hid); if (!SUCCESS(ret)) { return ret; } } for (uint64 i = 0; i < num_insert_threads; i++) { - platform_thread_join(params[i].thread); + platform_thread_join(¶ms[i].thread); } task_perform_until_quiescent(ts); @@ -1601,19 +1588,17 @@ test_splinter_periodic(system_config *cfg, */ for (uint64 i = 0; i < num_insert_threads; i++) { platform_status ret; - ret = task_thread_create("insert_thread", - test_trunk_insert_thread, - ¶ms[i], - core_get_scratch_size(), - ts, - hid, - ¶ms[i].thread); + ret = platform_thread_create(¶ms[i].thread, + FALSE, + test_trunk_insert_thread, + ¶ms[i], + hid); if (!SUCCESS(ret)) { return ret; } } for (uint64 i = 0; i < num_insert_threads; i++) { - platform_thread_join(params[i].thread); + platform_thread_join(¶ms[i].thread); } task_perform_until_quiescent(ts); @@ -1675,19 +1660,17 @@ test_splinter_periodic(system_config *cfg, async_ctxt_init( hid, max_async_inflight, ¶ms[i].async_lookup[j]); } - ret = task_thread_create("lookup thread", - test_trunk_lookup_thread, - ¶ms[i], - core_get_scratch_size(), - ts, - hid, - ¶ms[i].thread); + ret = platform_thread_create(¶ms[i].thread, + FALSE, + test_trunk_lookup_thread, + ¶ms[i], + hid); if (!SUCCESS(ret)) { return ret; } } for (uint64 i = 0; i < num_lookup_threads; i++) { - platform_thread_join(params[i].thread); + platform_thread_join(¶ms[i].thread); } total_time = platform_timestamp_elapsed(start_time); @@ -1761,19 +1744,14 @@ test_splinter_periodic(system_config *cfg, for (uint64 i = 0; i < num_range_threads; i++) { platform_status ret; - ret = task_thread_create("range thread", - test_trunk_range_thread, - ¶ms[i], - core_get_scratch_size(), - ts, - hid, - ¶ms[i].thread); + ret = platform_thread_create( + ¶ms[i].thread, FALSE, test_trunk_range_thread, ¶ms[i], hid); if (!SUCCESS(ret)) { return ret; } } for (uint64 i = 0; i < num_range_threads; i++) { - platform_thread_join(params[i].thread); + platform_thread_join(¶ms[i].thread); } total_time = platform_timestamp_elapsed(start_time); @@ -1818,19 +1796,14 @@ test_splinter_periodic(system_config *cfg, for (uint64 i = 0; i < num_range_threads; i++) { platform_status ret; - ret = task_thread_create("range thread", - test_trunk_range_thread, - ¶ms[i], - core_get_scratch_size(), - ts, - hid, - ¶ms[i].thread); + ret = platform_thread_create( + ¶ms[i].thread, FALSE, test_trunk_range_thread, ¶ms[i], hid); if (!SUCCESS(ret)) { return ret; } } for (uint64 i = 0; i < num_range_threads; i++) - platform_thread_join(params[i].thread); + platform_thread_join(¶ms[i].thread); total_time = platform_timestamp_elapsed(start_time); @@ -1874,19 +1847,14 @@ test_splinter_periodic(system_config *cfg, for (uint64 i = 0; i < num_range_threads; i++) { platform_status ret; - ret = task_thread_create("range thread", - test_trunk_range_thread, - ¶ms[i], - core_get_scratch_size(), - ts, - hid, - ¶ms[i].thread); + ret = platform_thread_create( + ¶ms[i].thread, FALSE, test_trunk_range_thread, ¶ms[i], hid); if (!SUCCESS(ret)) { return ret; } } for (uint64 i = 0; i < num_range_threads; i++) - platform_thread_join(params[i].thread); + platform_thread_join(¶ms[i].thread); total_time = platform_timestamp_elapsed(start_time); @@ -2034,7 +2002,7 @@ test_splinter_parallel_perf(system_config *cfg, // Wait-for threads to finish ... for (uint64 i = 0; i < num_threads; i++) { - platform_thread_join(params[i].thread); + platform_thread_join(¶ms[i].thread); } uint64 total_time = platform_timestamp_elapsed(start_time); @@ -2203,19 +2171,14 @@ test_splinter_delete(system_config *cfg, // Insert: round 1 for (uint64 i = 0; i < num_insert_threads; i++) { platform_status ret; - ret = task_thread_create("insert thread", - test_trunk_insert_thread, - ¶ms[i], - core_get_scratch_size(), - ts, - hid, - ¶ms[i].thread); + ret = platform_thread_create( + ¶ms[i].thread, FALSE, test_trunk_insert_thread, ¶ms[i], hid); if (!SUCCESS(ret)) { return ret; } } for (uint64 i = 0; i < num_insert_threads; i++) { - platform_thread_join(params[i].thread); + platform_thread_join(¶ms[i].thread); } total_time = platform_timestamp_elapsed(start_time); @@ -2247,19 +2210,14 @@ test_splinter_delete(system_config *cfg, start_time = platform_get_timestamp(); for (uint64 i = 0; i < num_insert_threads; i++) { platform_status ret; - ret = task_thread_create("delete thread", - test_trunk_insert_thread, - ¶ms[i], - core_get_scratch_size(), - ts, - hid, - ¶ms[i].thread); + ret = platform_thread_create( + ¶ms[i].thread, FALSE, test_trunk_insert_thread, ¶ms[i], hid); if (!SUCCESS(ret)) { return ret; } } for (uint64 i = 0; i < num_insert_threads; i++) - platform_thread_join(params[i].thread); + platform_thread_join(¶ms[i].thread); total_time = platform_timestamp_elapsed(start_time); platform_default_log( @@ -2294,22 +2252,17 @@ test_splinter_delete(system_config *cfg, for (uint8 j = 0; j < num_tables; j++) { async_ctxt_init(hid, max_async_inflight, ¶ms[i].async_lookup[j]); } - rc = task_thread_create("lookup thread", - test_trunk_lookup_thread, - ¶ms[i], - core_get_scratch_size(), - ts, - hid, - ¶ms[i].thread); + rc = platform_thread_create( + ¶ms[i].thread, FALSE, test_trunk_lookup_thread, ¶ms[i], hid); if (!SUCCESS(rc)) { for (uint64 j = 0; j < i; j++) { - platform_thread_join(params[i].thread); + platform_thread_join(¶ms[i].thread); } goto destroy_splinter; } } for (uint64 i = 0; i < num_lookup_threads; i++) { - platform_thread_join(params[i].thread); + platform_thread_join(¶ms[i].thread); } total_time = platform_timestamp_elapsed(start_time); @@ -2500,6 +2453,8 @@ splinter_test(int argc, char *argv[]) test_exec_config test_exec_cfg; ZERO_STRUCT(test_exec_cfg); + platform_register_thread(); + // Defaults num_insert_threads = num_lookup_threads = num_range_lookup_threads = 1; max_async_inflight = 64; @@ -2742,26 +2697,23 @@ splinter_test(int argc, char *argv[]) system_cfg[0].io_cfg.kernel_queue_size); } - platform_io_handle *io = TYPED_MALLOC(hid, io); - platform_assert(io != NULL); - rc = io_handle_init(io, &system_cfg[0].io_cfg, hid); - if (!SUCCESS(rc)) { - goto io_free; + io_handle *io = io_handle_create(&system_cfg[0].io_cfg, hid); + if (io == NULL) { + platform_error_log("Failed to create IO handle\n"); + rc = STATUS_NO_MEMORY; + goto cfg_free; } - rc = test_init_task_system(hid, io, &ts, &system_cfg[0].task_cfg); + rc = test_init_task_system(hid, &ts, &system_cfg[0].task_cfg); if (!SUCCESS(rc)) { platform_error_log("Failed to init splinter state: %s\n", platform_status_to_string(rc)); - goto handle_deinit; + goto handle_destroy; } rc_allocator al; - rc_allocator_init(&al, - &system_cfg[0].allocator_cfg, - (io_handle *)io, - hid, - platform_get_module_id()); + rc_allocator_init( + &al, &system_cfg[0].allocator_cfg, io, hid, platform_get_module_id()); platform_error_log("Running splinter_test with %d caches\n", num_caches); clockcache *cc = TYPED_ARRAY_MALLOC(hid, cc, num_caches); @@ -2769,7 +2721,7 @@ splinter_test(int argc, char *argv[]) for (uint8 idx = 0; idx < num_caches; idx++) { rc = clockcache_init(&cc[idx], &system_cfg[idx].cache_cfg, - (io_handle *)io, + io, (allocator *)&al, "test", hid, @@ -2894,7 +2846,7 @@ splinter_test(int argc, char *argv[]) test_data_config->key_to_string; } rc = test_functionality(alp, - (io_handle *)io, + io, caches, system_cfg, seed, @@ -2928,15 +2880,13 @@ splinter_test(int argc, char *argv[]) allocator_assert_noleaks(alp); rc_allocator_deinit(&al); test_deinit_task_system(hid, &ts); -handle_deinit: - io_handle_deinit(io); -io_free: - platform_free(hid, io); +handle_destroy: + io_handle_destroy(io); cfg_free: platform_free(hid, system_cfg); platform_free(hid, test_cfg); heap_destroy: platform_heap_destroy(&hid); - + platform_deregister_thread(); return SUCCESS(rc) ? 0 : -1; } diff --git a/tests/functional/splinter_test.h b/tests/functional/splinter_test.h index 5511a54d2..89956fb92 100644 --- a/tests/functional/splinter_test.h +++ b/tests/functional/splinter_test.h @@ -1,4 +1,4 @@ -// Copyright 2018-2021 VMware, Inc. +// Copyright 2018-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* @@ -8,8 +8,7 @@ * splinter tests. */ -#ifndef _SPLINTER_TEST_H_ -#define _SPLINTER_TEST_H_ +#pragma once #include "../config.h" #include "test.h" @@ -141,5 +140,3 @@ test_config_parse(test_config *cfg, platform_error_log("\t--key-type (rand, seq, semiseq)\n"); platform_error_log("\t--semiseq-freq\n"); } - -#endif /* _SPLINTER_TEST_H_ */ diff --git a/tests/functional/test.h b/tests/functional/test.h index 62db05ca5..a8363faef 100644 --- a/tests/functional/test.h +++ b/tests/functional/test.h @@ -1,4 +1,4 @@ -// Copyright 2018-2021 VMware, Inc. +// Copyright 2018-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* @@ -9,11 +9,9 @@ #pragma once -#include "cache.h" #include "clockcache.h" #include "../config.h" #include "splinterdb/data.h" -#include "rc_allocator.h" #include "shard_log.h" #include "core.h" #include "../test_data.h" @@ -57,12 +55,11 @@ splinter_io_apis_test(int argc, char *argv[]); */ static inline platform_status test_init_task_system(platform_heap_id hid, - platform_io_handle *ioh, task_system **system, const task_system_config *cfg) { // splinter initialization - return task_system_create(hid, ioh, system, cfg); + return task_system_create(hid, system, cfg); } static inline void @@ -253,10 +250,8 @@ test_config_init(system_config *system_cfg, // OUT uint64 num_bg_threads[NUM_TASK_TYPES] = {0}; num_bg_threads[TASK_TYPE_NORMAL] = master_cfg->num_normal_bg_threads; num_bg_threads[TASK_TYPE_MEMTABLE] = master_cfg->num_memtable_bg_threads; - platform_status rc = task_system_config_init(&system_cfg->task_cfg, - master_cfg->use_stats, - num_bg_threads, - core_get_scratch_size()); + platform_status rc = task_system_config_init( + &system_cfg->task_cfg, master_cfg->use_stats, num_bg_threads); platform_assert_status_ok(rc); rc = routing_config_init(&system_cfg->filter_cfg, diff --git a/tests/functional/test_async.c b/tests/functional/test_async.c index 7cbae5c4a..d4c4f9342 100644 --- a/tests/functional/test_async.c +++ b/tests/functional/test_async.c @@ -1,4 +1,4 @@ -// Copyright 2018-2021 VMware, Inc. +// Copyright 2018-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* diff --git a/tests/functional/test_async.h b/tests/functional/test_async.h index 9193ef696..03b3e79d9 100644 --- a/tests/functional/test_async.h +++ b/tests/functional/test_async.h @@ -1,4 +1,4 @@ -// Copyright 2018-2021 VMware, Inc. +// Copyright 2018-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* diff --git a/tests/functional/test_dispatcher.c b/tests/functional/test_dispatcher.c index 354983058..1f051033b 100644 --- a/tests/functional/test_dispatcher.c +++ b/tests/functional/test_dispatcher.c @@ -1,4 +1,4 @@ -// Copyright 2018-2021 VMware, Inc. +// Copyright 2018-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 #include "platform.h" diff --git a/tests/functional/test_functionality.c b/tests/functional/test_functionality.c index 976a57bf9..4fd98f990 100644 --- a/tests/functional/test_functionality.c +++ b/tests/functional/test_functionality.c @@ -1,4 +1,4 @@ -// Copyright 2018-2021 VMware, Inc. +// Copyright 2018-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 #include "platform.h" diff --git a/tests/functional/test_functionality.h b/tests/functional/test_functionality.h index b219cb4ca..e112bb311 100644 --- a/tests/functional/test_functionality.h +++ b/tests/functional/test_functionality.h @@ -1,4 +1,4 @@ -// Copyright 2018-2021 VMware, Inc. +// Copyright 2018-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 #include "allocator.h" diff --git a/tests/functional/test_splinter_shadow.c b/tests/functional/test_splinter_shadow.c index 228cec1e7..f1be8b20c 100644 --- a/tests/functional/test_splinter_shadow.c +++ b/tests/functional/test_splinter_shadow.c @@ -1,4 +1,4 @@ -// Copyright 2018-2021 VMware, Inc. +// Copyright 2018-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* diff --git a/tests/functional/test_splinter_shadow.h b/tests/functional/test_splinter_shadow.h index 4dcc2da6d..7b78d6600 100644 --- a/tests/functional/test_splinter_shadow.h +++ b/tests/functional/test_splinter_shadow.h @@ -1,4 +1,4 @@ -// Copyright 2018-2021 VMware, Inc. +// Copyright 2018-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* diff --git a/tests/functional/ycsb_test.c b/tests/functional/ycsb_test.c index 6961acd86..98864b268 100644 --- a/tests/functional/ycsb_test.c +++ b/tests/functional/ycsb_test.c @@ -1,8 +1,6 @@ -// Copyright 2018-2021 VMware, Inc. +// Copyright 2018-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 -#include "platform.h" - #include "core.h" #include "task.h" #include "rc_allocator.h" @@ -447,13 +445,8 @@ run_ycsb_phase(core_handle *spl, int j; for (j = 0; j < phase->params[i].nthreads; j++) { platform_assert(cur_thread < nthreads); - ret = task_thread_create("ycsb_thread", - ycsb_thread, - &phase->params[i], - core_get_scratch_size(), - ts, - hid, - &threads[cur_thread]); + ret = platform_thread_create( + &threads[cur_thread], FALSE, ycsb_thread, &phase->params[i], hid); if (!SUCCESS(ret)) { success = -1; goto shutdown; @@ -464,7 +457,7 @@ run_ycsb_phase(core_handle *spl, shutdown: while (0 < nthreads) { - platform_status result = platform_thread_join(threads[nthreads - 1]); + platform_status result = platform_thread_join(&threads[nthreads - 1]); if (!SUCCESS(result)) { success = -1; break; @@ -758,7 +751,7 @@ load_ycsb_logs(int argc, platform_assert(start_line == num_lines); for (uint64 i = 0; i < num_threads; i++) { - platform_thread_join(params[i].thread); + platform_thread_join(¶ms[i].thread); if (params[i].ycsb_ops == NULL) { platform_error_log("Bad log file: %s\n", params[i].filename); goto bad_params; @@ -1160,6 +1153,8 @@ ycsb_test(int argc, char *argv[]) int args_consumed; test_message_generator gen; + platform_register_thread(); + uint64 log_size_bytes, memory_bytes; rc = load_ycsb_logs(argc, argv, @@ -1171,6 +1166,7 @@ ycsb_test(int argc, char *argv[]) &memory_bytes); if (!SUCCESS(rc) || phases == NULL) { platform_default_log("Failed to load ycsb logs\n"); + platform_deregister_thread(); return -1; } platform_default_log("Log size: %luMiB\n", B_TO_MiB(log_size_bytes)); @@ -1260,21 +1256,18 @@ ycsb_test(int argc, char *argv[]) // platform_assert(sys_rc == 0); // platform_free(hid, resize_hugetlb_command); - platform_io_handle *io = TYPED_MALLOC(hid, io); - platform_assert(io != NULL); - if (!SUCCESS(rc)) { - goto free_iohandle; - } - rc = io_handle_init(io, &system_cfg->io_cfg, hid); - if (!SUCCESS(rc)) { - goto free_iohandle; + io_handle *io = io_handle_create(&system_cfg->io_cfg, hid); + if (io == NULL) { + platform_error_log("Failed to create IO handle\n"); + rc = STATUS_NO_MEMORY; + goto cleanup; } - rc = test_init_task_system(hid, io, &ts, &task_cfg); + rc = test_init_task_system(hid, &ts, &task_cfg); if (!SUCCESS(rc)) { platform_error_log("Failed to init splinter state: %s\n", platform_status_to_string(rc)); - goto deinit_iohandle; + goto destroy_iohandle; } rc_allocator al; @@ -1282,14 +1275,11 @@ ycsb_test(int argc, char *argv[]) core_handle *spl; if (use_existing) { - rc_allocator_mount(&al, - &system_cfg->allocator_cfg, - (io_handle *)io, - hid, - platform_get_module_id()); + rc_allocator_mount( + &al, &system_cfg->allocator_cfg, io, hid, platform_get_module_id()); rc = clockcache_init(cc, &system_cfg->cache_cfg, - (io_handle *)io, + io, (allocator *)&al, "test", hid, @@ -1303,14 +1293,11 @@ ycsb_test(int argc, char *argv[]) hid); platform_assert(spl); } else { - rc_allocator_init(&al, - &system_cfg->allocator_cfg, - (io_handle *)io, - hid, - platform_get_module_id()); + rc_allocator_init( + &al, &system_cfg->allocator_cfg, io, hid, platform_get_module_id()); rc = clockcache_init(cc, &system_cfg->cache_cfg, - (io_handle *)io, + io, (allocator *)&al, "test", hid, @@ -1352,13 +1339,11 @@ ycsb_test(int argc, char *argv[]) } platform_free(hid, phases); -deinit_iohandle: - io_handle_deinit(io); -free_iohandle: - platform_free(hid, io); +destroy_iohandle: + io_handle_destroy(io); cleanup: platform_free(hid, system_cfg); platform_heap_destroy(&hid); - + platform_deregister_thread(); return SUCCESS(rc) ? 0 : -1; } diff --git a/tests/test_common.c b/tests/test_common.c index 8e81d0c94..c41e12e2b 100644 --- a/tests/test_common.c +++ b/tests/test_common.c @@ -1,4 +1,4 @@ -// Copyright 2022 VMware, Inc. +// Copyright 2022-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* @@ -8,7 +8,6 @@ * Module contains functions shared between functional/ and unit/ test sources. * ----------------------------------------------------------------------------- */ -#include "splinterdb/public_platform.h" #include "core.h" #include "functional/test.h" #include "functional/test_async.h" @@ -158,7 +157,7 @@ trace_wait_for_gdb(void) if (!gdb_msg_printed) { platform_default_log( "Looping ... Attach gdb to OS-pid=%d; Set breakpoint in %s_hook\n", - platform_getpid(), + platform_get_os_pid(), __func__); gdb_msg_printed = TRUE; } diff --git a/tests/test_common.h b/tests/test_common.h index c359cffe5..017af6e57 100644 --- a/tests/test_common.h +++ b/tests/test_common.h @@ -1,4 +1,4 @@ -// Copyright 2022 VMware, Inc. +// Copyright 2022-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* diff --git a/tests/test_data.c b/tests/test_data.c index 9467c4bd3..40212b6e1 100644 --- a/tests/test_data.c +++ b/tests/test_data.c @@ -1,8 +1,8 @@ -// Copyright 2018-2021 VMware, Inc. +// Copyright 2018-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 #include "test_data.h" -#include "data_internal.h" +#include "platform_hash.h" typedef struct data_test_config { data_config super; diff --git a/tests/test_data.h b/tests/test_data.h index b15f7b600..675376a60 100644 --- a/tests/test_data.h +++ b/tests/test_data.h @@ -1,8 +1,9 @@ -// Copyright 2018-2021 VMware, Inc. +// Copyright 2018-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 #pragma once +#include "platform_log.h" #include "splinterdb/data.h" #include "util.h" diff --git a/tests/unit/btree_stress_test.c b/tests/unit/btree_stress_test.c index 6c069641e..48588f12d 100644 --- a/tests/unit/btree_stress_test.c +++ b/tests/unit/btree_stress_test.c @@ -1,4 +1,4 @@ -// Copyright 2021 VMware, Inc. +// Copyright 2021-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* @@ -12,14 +12,14 @@ #include #include -#include "splinterdb/public_platform.h" #include "unit_tests.h" #include "ctest.h" // This is required for all test-case files. #include "functional/test.h" #include "splinterdb/data.h" #include "../config.h" -#include "io.h" +#include "platform_io.h" +#include "platform_units.h" #include "rc_allocator.h" #include "clockcache.h" #include "btree_private.h" @@ -111,15 +111,16 @@ CTEST_DATA(btree_stress) platform_heap_id hid; // Stuff needed to setup and exercise multiple threads. - platform_io_handle io; - task_system *ts; - rc_allocator al; - clockcache cc; + io_handle *io; + task_system *ts; + rc_allocator al; + clockcache cc; }; // Setup function for suite, called before every test in suite CTEST_SETUP(btree_stress) { + platform_register_thread(); config_set_defaults(&data->master_cfg); data->master_cfg.cache_capacity = GiB_TO_B(5); data->data_cfg = test_data_config; @@ -137,8 +138,8 @@ CTEST_SETUP(btree_stress) &data->master_cfg, &data->cache_cfg.super, data->data_cfg) - || !init_task_config_from_master_config( - &data->task_cfg, &data->master_cfg, sizeof(btree_scratch))) + || !init_task_config_from_master_config(&data->task_cfg, + &data->master_cfg)) { ASSERT_TRUE(FALSE, "Failed to parse args\n"); } @@ -153,25 +154,25 @@ CTEST_SETUP(btree_stress) } // Setup execution of concurrent threads data->ts = NULL; - if (!SUCCESS(io_handle_init(&data->io, &data->io_cfg, data->hid)) - || !SUCCESS( - task_system_create(data->hid, &data->io, &data->ts, &data->task_cfg)) + data->io = io_handle_create(&data->io_cfg, data->hid); + if (data->io == NULL + || !SUCCESS(task_system_create(data->hid, &data->ts, &data->task_cfg)) || !SUCCESS(rc_allocator_init(&data->al, &data->allocator_cfg, - (io_handle *)&data->io, + data->io, data->hid, platform_get_module_id())) || !SUCCESS(clockcache_init(&data->cc, &data->cache_cfg, - (io_handle *)&data->io, + data->io, (allocator *)&data->al, "test", data->hid, platform_get_module_id()))) { - ASSERT_TRUE( - FALSE, - "Failed to init io or task system or rc_allocator or clockcache\n"); + ASSERT_TRUE(FALSE, + "Failed to create io or init task system or rc_allocator or " + "clockcache\n"); } } @@ -181,8 +182,9 @@ CTEST_TEARDOWN(btree_stress) clockcache_deinit(&data->cc); rc_allocator_deinit(&data->al); task_system_destroy(data->hid, &data->ts); - io_handle_deinit(&data->io); + io_handle_destroy(data->io); platform_heap_destroy(&data->hid); + platform_deregister_thread(); } CTEST2(btree_stress, iterator_basics) @@ -262,20 +264,15 @@ CTEST2(btree_stress, test_random_inserts_concurrent) } for (uint64 i = 0; i < nthreads; i++) { - platform_status ret = task_thread_create("insert thread", - insert_thread, - ¶ms[i], - 0, - data->ts, - data->hid, - &threads[i]); + platform_status ret = platform_thread_create( + &threads[i], FALSE, insert_thread, ¶ms[i], data->hid); ASSERT_TRUE(SUCCESS(ret)); // insert_tests((cache *)&cc, &dbtree_cfg, &test_scratch, &mini, // root_addr, 0, nkvs); } for (uint64 thread_no = 0; thread_no < nthreads; thread_no++) { - platform_thread_join(threads[thread_no]); + platform_thread_join(&threads[thread_no]); } int rc = query_tests((cache *)&data->cc, diff --git a/tests/unit/btree_test.c b/tests/unit/btree_test.c index 702826ee6..629925114 100644 --- a/tests/unit/btree_test.c +++ b/tests/unit/btree_test.c @@ -1,4 +1,4 @@ -// Copyright 2021 VMware, Inc. +// Copyright 2021-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* @@ -9,14 +9,13 @@ * files. Validates correctness of variable key-value size support in BTree. * ----------------------------------------------------------------------------- */ -#include "splinterdb/public_platform.h" #include "unit_tests.h" #include "ctest.h" // This is required for all test-case files. #include "test_data.h" #include "splinterdb/data.h" -#include "io.h" -#include "rc_allocator.h" +#include "platform_io.h" +#include "platform_units.h" #include "clockcache.h" #include "btree_private.h" #include "btree_test_common.h" @@ -75,6 +74,7 @@ CTEST_DATA(btree) // Optional setup function for suite, called before every test in suite CTEST_SETUP(btree) { + platform_register_thread(); config_set_defaults(&data->master_cfg); uint64 heap_capacity = (1 * GiB); @@ -112,6 +112,7 @@ CTEST_SETUP(btree) CTEST_TEARDOWN(btree) { platform_heap_destroy(&data->hid); + platform_deregister_thread(); } /* diff --git a/tests/unit/btree_test_common.c b/tests/unit/btree_test_common.c index 4f7590825..6709b93e3 100644 --- a/tests/unit/btree_test_common.c +++ b/tests/unit/btree_test_common.c @@ -1,4 +1,4 @@ -// Copyright 2021 VMware, Inc. +// Copyright 2021-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* * btree_test_common.c: @@ -51,16 +51,15 @@ init_clockcache_config_from_master_config(clockcache_config *cache_cfg, int init_task_config_from_master_config(task_system_config *task_cfg, - const master_config *master_cfg, - uint64 scratch_size) + const master_config *master_cfg) { platform_status rc; uint64 num_bg_threads[NUM_TASK_TYPES] = {0}; num_bg_threads[TASK_TYPE_NORMAL] = master_cfg->num_normal_bg_threads; num_bg_threads[TASK_TYPE_MEMTABLE] = master_cfg->num_memtable_bg_threads; - rc = task_system_config_init( - task_cfg, master_cfg->use_stats, num_bg_threads, scratch_size); + rc = + task_system_config_init(task_cfg, master_cfg->use_stats, num_bg_threads); return SUCCESS(rc); } diff --git a/tests/unit/btree_test_common.h b/tests/unit/btree_test_common.h index 26a54757e..a2e5ac4fd 100644 --- a/tests/unit/btree_test_common.h +++ b/tests/unit/btree_test_common.h @@ -1,4 +1,4 @@ -// Copyright 2021 VMware, Inc. +// Copyright 2021-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* * btree_test_common.h: Shared header file with protoypes etc. used @@ -8,8 +8,7 @@ #pragma once #include "../config.h" -#include "io.h" -#include "rc_allocator.h" +#include "platform_io.h" #include "clockcache.h" #include "task.h" #include "btree.h" @@ -35,8 +34,7 @@ init_clockcache_config_from_master_config(clockcache_config *cache_cfg, int init_task_config_from_master_config(task_system_config *task_cfg, - const master_config *master_cfg, - uint64 scratch_size); + const master_config *master_cfg); int init_btree_config_from_master_config(btree_config *dbtree_cfg, diff --git a/tests/unit/config_parse_test.c b/tests/unit/config_parse_test.c index 6f2bd2705..9a728e7df 100644 --- a/tests/unit/config_parse_test.c +++ b/tests/unit/config_parse_test.c @@ -1,4 +1,4 @@ -// Copyright 2022 VMware, Inc. +// Copyright 2022-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* @@ -14,15 +14,8 @@ * checking for the result of the config-parsing routines. * ----------------------------------------------------------------------------- */ -#include "splinterdb/public_platform.h" #include "core.h" -#include "clockcache.h" -#include "allocator.h" -#include "rc_allocator.h" -#include "shard_log.h" -#include "task.h" #include "functional/test.h" -#include "functional/test_async.h" #include "test_common.h" #include "unit_tests.h" #include "ctest.h" // This is required for all test-case files. @@ -40,6 +33,7 @@ CTEST_DATA(config_parse) // Optional setup function for suite, called before every test in suite CTEST_SETUP(config_parse) { + platform_register_thread(); uint64 heap_capacity = (1024 * MiB); // Create a heap for io, allocator, cache and splinter platform_status rc = platform_heap_create( @@ -53,6 +47,7 @@ CTEST_SETUP(config_parse) CTEST_TEARDOWN(config_parse) { platform_heap_destroy(&data->hid); + platform_deregister_thread(); } /* diff --git a/tests/unit/ctest.h b/tests/unit/ctest.h index 1c88b30fc..c80202b23 100644 --- a/tests/unit/ctest.h +++ b/tests/unit/ctest.h @@ -1,5 +1,5 @@ /* Copyright 2011-2021 Bas van den Berg - * Copyright 2018-2021 VMware, Inc. + * Copyright 2018-2026 VMware, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/tests/unit/large_inserts_stress_test.c b/tests/unit/large_inserts_stress_test.c index c27b500c4..d52ad9d0e 100644 --- a/tests/unit/large_inserts_stress_test.c +++ b/tests/unit/large_inserts_stress_test.c @@ -1,4 +1,4 @@ -// Copyright 2022 VMware, Inc. +// Copyright 2022-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* @@ -14,9 +14,13 @@ #include #include -#include "platform.h" -#include "splinterdb/public_platform.h" +#include "platform_threads.h" +#include "platform_units.h" +#include "platform_typed_alloc.h" +#include "platform_log.h" +#include "platform_time.h" #include "splinterdb/default_data_config.h" +#include "splinterdb/platform_linux/public_platform.h" #include "splinterdb/splinterdb.h" #include "config.h" #include "unit_tests.h" @@ -39,11 +43,10 @@ typedef struct { uint64 num_insert_threads; int random_key_fd; // Options to choose the type of key inserted int random_val_fd; // Options to choose the type of value inserted - bool is_thread; // Is main() or thread executing worker fn } worker_config; // Function Prototypes -static void * +static void exec_worker_thread(void *w); static void @@ -98,9 +101,10 @@ CTEST_DATA(large_inserts_stress) // Optional setup function for suite, called before every test in suite CTEST_SETUP(large_inserts_stress) { + platform_register_thread(); // First, register that main() is being run as a parent process data->am_parent = TRUE; - data->this_pid = platform_getpid(); + data->this_pid = platform_get_os_pid(); platform_status rc; uint64 heap_capacity = (64 * MiB); // small heap is sufficient. @@ -162,6 +166,7 @@ CTEST_TEARDOWN(large_inserts_stress) splinterdb_close(&data->kvsb); platform_heap_destroy(&data->hid); } + platform_deregister_thread(); } /* @@ -314,7 +319,7 @@ CTEST2(large_inserts_stress, test_seq_key_seq_values_inserts_forked) wcfg.master_cfg = &data->master_cfg; wcfg.num_inserts = data->num_inserts; - int pid = platform_getpid(); + int pid = platform_get_os_pid(); if (wcfg.master_cfg->fork_child) { pid = fork(); @@ -325,7 +330,7 @@ CTEST2(large_inserts_stress, test_seq_key_seq_values_inserts_forked) } else if (pid) { platform_default_log("OS-pid=%d, Thread-ID=%lu: " "Waiting for child pid=%d to complete ...\n", - platform_getpid(), + platform_get_os_pid(), platform_get_tid(), pid); @@ -335,20 +340,20 @@ CTEST2(large_inserts_stress, test_seq_key_seq_values_inserts_forked) "Child execution wait() completed." " Resuming parent ...\n", platform_get_tid(), - platform_getpid()); + platform_get_os_pid()); } } if (pid == 0) { + platform_register_thread(); // Record in global data that we are now running as a child. data->am_parent = FALSE; - data->this_pid = platform_getpid(); + data->this_pid = platform_get_os_pid(); platform_default_log( "OS-pid=%d Running as %s process ...\n", data->this_pid, (wcfg.master_cfg->fork_child ? "forked child" : "parent")); - splinterdb_register_thread(wcfg.kvsb); exec_worker_thread(&wcfg); @@ -356,7 +361,7 @@ CTEST2(large_inserts_stress, test_seq_key_seq_values_inserts_forked) ", completed inserts.\n", data->this_pid, platform_get_tid()); - splinterdb_deregister_thread(wcfg.kvsb); + platform_deregister_thread(); exit(0); return; } @@ -554,7 +559,6 @@ do_inserts_n_threads(splinterdb *kvsb, ((random_key_fd < 0) ? 0 : (wcfg[ictr].num_inserts * ictr)); wcfg[ictr].random_key_fd = random_key_fd; wcfg[ictr].random_val_fd = random_val_fd; - wcfg[ictr].is_thread = TRUE; } platform_thread *thread_ids = @@ -562,24 +566,15 @@ do_inserts_n_threads(splinterdb *kvsb, // Fire-off the threads to drive inserts ... for (int tctr = 0; tctr < num_insert_threads; tctr++) { - int rc = pthread_create( - &thread_ids[tctr], NULL, &exec_worker_thread, &wcfg[tctr]); - ASSERT_EQUAL(0, rc); + platform_status rc = platform_thread_create( + &thread_ids[tctr], FALSE, &exec_worker_thread, &wcfg[tctr], hid); + platform_assert_status_ok(rc); } // Wait for all threads to complete ... for (int tctr = 0; tctr < num_insert_threads; tctr++) { - void *thread_rc; - int rc = pthread_join(thread_ids[tctr], &thread_rc); - ASSERT_EQUAL(0, rc); - if (thread_rc != 0) { - fprintf(stderr, - "Thread %d [ID=%lu] had error: %p\n", - tctr, - thread_ids[tctr], - thread_rc); - ASSERT_TRUE(FALSE); - } + platform_status rc = platform_thread_join(&thread_ids[tctr]); + platform_assert_status_ok(rc); } platform_free(hid, thread_ids); platform_free(hid, wcfg); @@ -595,14 +590,14 @@ do_inserts_n_threads(splinterdb *kvsb, * to be inserted. Can also choose whether value will be fully-packed. * ---------------------------------------------------------------------------- */ -static void * +static void exec_worker_thread(void *w) { + worker_config *wcfg = (worker_config *)w; + char key_data[TEST_KEY_SIZE]; char val_data[TEST_VALUE_SIZE]; - worker_config *wcfg = (worker_config *)w; - splinterdb *kvsb = wcfg->kvsb; uint64 start_key = wcfg->start_value; uint64 num_inserts = wcfg->num_inserts; @@ -611,9 +606,6 @@ exec_worker_thread(void *w) uint64 start_time = platform_get_timestamp(); - if (wcfg->is_thread) { - splinterdb_register_thread(kvsb); - } threadid thread_idx = platform_get_tid(); // Test is written to insert multiples of millions per thread. @@ -682,7 +674,7 @@ exec_worker_thread(void *w) platform_default_log("OS-pid=%d, Thread-ID=%lu" ", Insert random value of " "fixed-length=%lu bytes.\n", - platform_getpid(), + platform_get_os_pid(), thread_idx, val_len); val_length_msg_printed = TRUE; @@ -696,7 +688,7 @@ exec_worker_thread(void *w) platform_default_log("OS-pid=%d, Thread-ID=%lu" ", Insert small-width sequential values of " "different lengths.\n", - platform_getpid(), + platform_get_os_pid(), thread_idx); val_length_msg_printed = TRUE; } @@ -705,7 +697,7 @@ exec_worker_thread(void *w) platform_default_log("OS-pid=%d, Thread-ID=%lu" ", Insert fully-packed fixed value of " "length=%lu bytes.\n", - platform_getpid(), + platform_get_os_pid(), thread_idx, val_len); val_length_msg_printed = TRUE; @@ -742,10 +734,4 @@ exec_worker_thread(void *w) ictr, // outer-loop ends at #-of-Millions inserted elapsed_s, (num_inserts / elapsed_s)); - - if (wcfg->is_thread) { - splinterdb_deregister_thread(kvsb); - } - - return 0; } diff --git a/tests/unit/limitations_test.c b/tests/unit/limitations_test.c index dec3cf77d..981b2a6df 100644 --- a/tests/unit/limitations_test.c +++ b/tests/unit/limitations_test.c @@ -1,4 +1,4 @@ -// Copyright 2022 VMware, Inc. +// Copyright 2022-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* @@ -11,17 +11,15 @@ * unworkable, or currently unsupported. * ----------------------------------------------------------------------------- */ -#include "splinterdb/public_platform.h" -#include "core.h" #include "clockcache.h" -#include "allocator.h" +#include "rc_allocator.h" #include "task.h" #include "functional/test.h" -#include "functional/test_async.h" #include "splinterdb/splinterdb.h" #include "splinterdb/default_data_config.h" #include "unit_tests.h" #include "ctest.h" // This is required for all test-case files. +#include "platform_units.h" #define TEST_MAX_KEY_SIZE 13 @@ -44,7 +42,7 @@ CTEST_DATA(limitations) rc_allocator al; // Following get setup pointing to allocated memory - platform_io_handle *io; + io_handle *io; clockcache *clock_cache; task_system *tasks; test_message_generator gen; @@ -60,6 +58,7 @@ CTEST_DATA(limitations) */ CTEST_SETUP(limitations) { + platform_register_thread(); // All test cases in this test usually deal with error handling set_log_streams_for_tests(MSG_LEVEL_ERRORS); @@ -78,6 +77,7 @@ CTEST_SETUP(limitations) CTEST_TEARDOWN(limitations) { platform_heap_destroy(&data->hid); + platform_deregister_thread(); } /* @@ -106,10 +106,6 @@ CTEST2(limitations, test_io_init_invalid_page_size) (char **)Ctest_argv); platform_assert_status_ok(rc); - // Allocate and initialize the IO sub-system. - data->io = TYPED_MALLOC(data->hid, data->io); - ASSERT_TRUE((data->io != NULL)); - // Hard-fix the configured default page-size to an illegal value uint64 page_size_configured = data->system_cfg->io_cfg.page_size; ASSERT_EQUAL(page_size_configured, 4096); @@ -117,23 +113,23 @@ CTEST2(limitations, test_io_init_invalid_page_size) data->system_cfg->io_cfg.page_size = 2048; // This should fail. - rc = io_handle_init(data->io, &data->system_cfg->io_cfg, data->hid); - ASSERT_FALSE(SUCCESS(rc)); + data->io = io_handle_create(&data->system_cfg->io_cfg, data->hid); + ASSERT_TRUE(data->io == NULL); // This should fail. data->system_cfg->io_cfg.page_size = (page_size_configured * 2); - rc = io_handle_init(data->io, &data->system_cfg->io_cfg, data->hid); - ASSERT_FALSE(SUCCESS(rc)); + data->io = io_handle_create(&data->system_cfg->io_cfg, data->hid); + ASSERT_TRUE(data->io == NULL); // Restore, and now set extent-size to invalid value data->system_cfg->io_cfg.page_size = page_size_configured; // This should succeed, finally!. - rc = io_handle_init(data->io, &data->system_cfg->io_cfg, data->hid); - ASSERT_TRUE(SUCCESS(rc)); + data->io = io_handle_create(&data->system_cfg->io_cfg, data->hid); + ASSERT_TRUE(data->io != NULL); // Release resources acquired in this test case. - io_handle_deinit(data->io); + io_handle_destroy(data->io); if (data->system_cfg) { platform_free(data->hid, data->system_cfg); @@ -164,10 +160,6 @@ CTEST2(limitations, test_io_init_invalid_extent_size) (char **)Ctest_argv); platform_assert_status_ok(rc); - // Allocate and initialize the IO sub-system. - data->io = TYPED_MALLOC(data->hid, data->io); - ASSERT_TRUE((data->io != NULL)); - uint64 pages_per_extent = (data->system_cfg->io_cfg.extent_size / data->system_cfg->io_cfg.page_size); ASSERT_EQUAL(MAX_PAGES_PER_EXTENT, @@ -180,26 +172,27 @@ CTEST2(limitations, test_io_init_invalid_extent_size) // This should fail. data->system_cfg->io_cfg.extent_size = data->system_cfg->io_cfg.page_size; - rc = io_handle_init(data->io, &data->system_cfg->io_cfg, data->hid); - ASSERT_FALSE(SUCCESS(rc)); + data->io = io_handle_create(&data->system_cfg->io_cfg, data->hid); + ASSERT_TRUE(data->io == NULL); // Halving the # of pages/extent. This should fail. data->system_cfg->io_cfg.extent_size = (data->system_cfg->io_cfg.page_size * pages_per_extent) / 2; - rc = io_handle_init(data->io, &data->system_cfg->io_cfg, data->hid); - ASSERT_FALSE(SUCCESS(rc)); + data->io = io_handle_create(&data->system_cfg->io_cfg, data->hid); + ASSERT_TRUE(data->io == NULL); // Doubling the # of pages/extent. This should fail. data->system_cfg->io_cfg.extent_size = (data->system_cfg->io_cfg.page_size * pages_per_extent * 2); - rc = io_handle_init(data->io, &data->system_cfg->io_cfg, data->hid); - ASSERT_FALSE(SUCCESS(rc)); + data->io = io_handle_create(&data->system_cfg->io_cfg, data->hid); + ASSERT_TRUE(data->io == NULL); data->system_cfg->io_cfg.extent_size = extent_size_configured; // This should succeed, finally!. - rc = io_handle_init(data->io, &data->system_cfg->io_cfg, data->hid); - ASSERT_TRUE(SUCCESS(rc)); + data->io = io_handle_create(&data->system_cfg->io_cfg, data->hid); + ASSERT_TRUE(data->io != NULL); + io_handle_destroy(data->io); if (data->system_cfg) { platform_free(data->hid, data->system_cfg); diff --git a/tests/unit/main.c b/tests/unit/main.c index 0b6ef2cf6..b6ed4801e 100644 --- a/tests/unit/main.c +++ b/tests/unit/main.c @@ -1,5 +1,5 @@ /* Copyright 2011-2021 Bas van den Berg - * Copyright 2021 VMware, Inc. + * Copyright 2021-2026 VMware, Inc. * SPDX-License-Identifier: Apache-2.0 */ #include @@ -22,6 +22,7 @@ #include "util.h" #include "ctest.h" #include "unit_tests.h" +#include "platform_threads.h" #define MSG_SIZE 4096 @@ -143,7 +144,7 @@ sighandler(int signum) * so it can terminate as expected */ signal(signum, SIG_DFL); - kill(platform_getpid(), signum); + kill(platform_get_os_pid(), signum); } #endif // CTEST_SEGFAULT @@ -329,7 +330,7 @@ ctest_main(int argc, const char *argv[]) } if (test->teardown && *test->teardown) (*test->teardown)(test->data); - // if we got here it's ok + // if we got here it's ok #ifdef CTEST_COLOR_OK color_print(ANSI_BGREEN, "[OK]"); #else diff --git a/tests/unit/misc_test.c b/tests/unit/misc_test.c index 53d04fd6a..0947b0c53 100644 --- a/tests/unit/misc_test.c +++ b/tests/unit/misc_test.c @@ -1,4 +1,4 @@ -// Copyright 2021 VMware, Inc. +// Copyright 2021-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* @@ -9,10 +9,10 @@ * ----------------------------------------------------------------------------- */ #include -#include "platform.h" #include "unit_tests.h" #include "ctest.h" // This is required for all test-case files. -#include "util.h" +#include "platform_log.h" +#include "platform_units.h" #define ASSERT_OUTBUF_LEN 200 @@ -67,13 +67,17 @@ CTEST_DATA(misc) CTEST_SETUP(misc) { + platform_register_thread(); // All test cases in this test usually deal with error handling set_log_streams_for_tests(MSG_LEVEL_ERRORS); data->log_output = platform_get_stdout_stream(); } // Optional teardown function for suite, called after every test in suite -CTEST_TEARDOWN(misc) {} +CTEST_TEARDOWN(misc) +{ + platform_deregister_thread(); +} /* * Basic test case that exercises assertion checking code with a message diff --git a/tests/unit/platform_apis_test.c b/tests/unit/platform_apis_test.c index 50df378c6..d7b49d4b3 100644 --- a/tests/unit/platform_apis_test.c +++ b/tests/unit/platform_apis_test.c @@ -1,4 +1,4 @@ -// Copyright 2023 VMware, Inc. +// Copyright 2023-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* @@ -16,9 +16,15 @@ #include #include "ctest.h" // This is required for all test-case files. -#include "platform.h" #include "config.h" #include "unit_tests.h" +#include "platform_semaphore.h" +#include "platform_threads.h" +#include "platform_units.h" +#include "platform_buffer.h" +#include "platform_spinlock.h" +#include "platform_mutex.h" +#include "platform_condvar.h" /* * Global data declaration macro: @@ -32,6 +38,7 @@ CTEST_DATA(platform_api) CTEST_SETUP(platform_api) { + platform_register_thread(); platform_status rc = STATUS_OK; bool use_shmem = config_parse_use_shmem(Ctest_argc, (char **)Ctest_argv); @@ -44,6 +51,7 @@ CTEST_SETUP(platform_api) CTEST_TEARDOWN(platform_api) { platform_heap_destroy(&data->hid); + platform_deregister_thread(); } /* @@ -114,10 +122,9 @@ CTEST2(platform_api, test_platform_semaphore_init_destroy) */ CTEST2(platform_api, test_platform_spinlock_init_destroy) { - platform_spinlock slock; - platform_module_id unused = 0; + platform_spinlock slock; - platform_status rc = platform_spinlock_init(&slock, unused, data->hid); + platform_status rc = platform_spinlock_init(&slock); ASSERT_TRUE(SUCCESS(rc)); rc = platform_spinlock_destroy(&slock); diff --git a/tests/unit/splinter_shmem_test.c b/tests/unit/splinter_shmem_test.c index eab78903b..7cb078cd7 100644 --- a/tests/unit/splinter_shmem_test.c +++ b/tests/unit/splinter_shmem_test.c @@ -1,4 +1,4 @@ -// Copyright 2021 VMware, Inc. +// Copyright 2021-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* @@ -8,8 +8,10 @@ * Exercises the interfaces in SplinterDB shared memory allocation module. * ----------------------------------------------------------------------------- */ -#include "splinterdb/public_platform.h" -#include "platform.h" +#include "platform_threads.h" +#include "platform_units.h" +#include "platform_typed_alloc.h" +#include "platform_log.h" #include "unit_tests.h" #include "ctest.h" // This is required for all test-case files. #include "shmem.h" @@ -59,6 +61,7 @@ CTEST_DATA(splinter_shmem) // By default, all test cases will deal with small shared memory segment. CTEST_SETUP(splinter_shmem) { + platform_register_thread(); data->shmem_capacity = (256 * MiB); // bytes platform_status rc = platform_heap_create( platform_get_module_id(), data->shmem_capacity, TRUE, &data->hid); @@ -72,6 +75,7 @@ CTEST_SETUP(splinter_shmem) CTEST_TEARDOWN(splinter_shmem) { platform_heap_destroy(&data->hid); + platform_deregister_thread(); } /* @@ -96,7 +100,7 @@ CTEST2(splinter_shmem, test_create_destroy_shmem) (requested - platform_shm_ctrlblock_size())); // Destroy shared memory and release memory. - platform_shmdestroy(&hid); + platform_shmdestroy((shmem_heap **)&hid); ASSERT_TRUE(hid == NULL); } @@ -381,7 +385,7 @@ CTEST2(splinter_shmem, test_concurrent_allocs_by_n_threads) tctr < ARRAY_SIZE(thread_cfg); tctr++, thread_cfgp++) { - rc = platform_thread_join(thread_cfgp->this_thread_id); + rc = platform_thread_join(&thread_cfgp->this_thread_id); ASSERT_TRUE(SUCCESS(rc)); } @@ -598,9 +602,6 @@ static void exec_thread_memalloc(void *arg) { thread_config *thread_cfg = (thread_config *)arg; - splinterdb *kvs = thread_cfg->splinter; - - splinterdb_register_thread(kvs); // Allocate a new memory fragment and connect head to output variable for // thread @@ -618,7 +619,6 @@ exec_thread_memalloc(void *arg) fragpp = &new_frag->next; nallocs++; } - splinterdb_deregister_thread(kvs); platform_default_log( "Thread-ID=%lu allocated %lu memory fragments of %lu bytes each.\n", diff --git a/tests/unit/splinter_test.c b/tests/unit/splinter_test.c index 864df461e..d9b8b4ff5 100644 --- a/tests/unit/splinter_test.c +++ b/tests/unit/splinter_test.c @@ -1,4 +1,4 @@ -// Copyright 2022 VMware, Inc. +// Copyright 2022-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* @@ -21,10 +21,10 @@ * $ bin/unit/splinter_test --memtable-capacity-mib 4 test_lookups * ----------------------------------------------------------------------------- */ -#include "splinterdb/public_platform.h" #include "core.h" #include "clockcache.h" #include "allocator.h" +#include "rc_allocator.h" #include "task.h" #include "functional/test.h" #include "functional/test_async.h" @@ -89,7 +89,7 @@ CTEST_DATA(splinter) // Following get setup pointing to allocated memory system_config *system_cfg; - platform_io_handle *io; + io_handle *io; clockcache *clock_cache; task_system *tasks; test_message_generator gen; @@ -106,6 +106,7 @@ CTEST_DATA(splinter) // clang-format off CTEST_SETUP(splinter) { + platform_register_thread(); bool use_shmem = config_parse_use_shmem(Ctest_argc, (char **)Ctest_argv); // Defaults: For basic unit-tests, use single threads @@ -158,17 +159,16 @@ CTEST_SETUP(splinter) } // Allocate and initialize the IO sub-system. - data->io = TYPED_MALLOC(data->hid, data->io); - ASSERT_TRUE((data->io != NULL)); - rc = io_handle_init(data->io, &data->system_cfg->io_cfg, data->hid); + data->io = io_handle_create(&data->system_cfg->io_cfg, data->hid); + ASSERT_TRUE((data->io != NULL), "Failed to create IO handle\n"); data->tasks = NULL; - rc = test_init_task_system(data->hid, data->io, &data->tasks, &data->system_cfg->task_cfg); + rc = test_init_task_system(data->hid, &data->tasks, &data->system_cfg->task_cfg); ASSERT_TRUE(SUCCESS(rc), "Failed to init splinter state: %s\n", platform_status_to_string(rc)); - rc_allocator_init(&data->al, &data->system_cfg->allocator_cfg, (io_handle *)data->io, data->hid, + rc_allocator_init(&data->al, &data->system_cfg->allocator_cfg, data->io, data->hid, platform_get_module_id()); data->clock_cache = TYPED_ARRAY_MALLOC(data->hid, data->clock_cache, num_caches); @@ -177,7 +177,7 @@ CTEST_SETUP(splinter) for (uint8 idx = 0; idx < num_caches; idx++) { rc = clockcache_init(&data->clock_cache[idx], &data->system_cfg[idx].cache_cfg, - (io_handle *)data->io, + data->io, (allocator *)&data->al, "test", data->hid, @@ -203,14 +203,14 @@ CTEST_TEARDOWN(splinter) rc_allocator_deinit(&data->al); test_deinit_task_system(data->hid, &data->tasks); - io_handle_deinit(data->io); - platform_free(data->hid, data->io); + io_handle_destroy(data->io); if (data->system_cfg) { platform_free(data->hid, data->system_cfg); } platform_heap_destroy(&data->hid); + platform_deregister_thread(); } /* diff --git a/tests/unit/splinterdb_forked_child_test.c b/tests/unit/splinterdb_forked_child_test.c index 01ef2e159..f5776155e 100644 --- a/tests/unit/splinterdb_forked_child_test.c +++ b/tests/unit/splinterdb_forked_child_test.c @@ -1,4 +1,4 @@ -// Copyright 2021 VMware, Inc. +// Copyright 2021-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* @@ -15,8 +15,6 @@ */ #include -#include "platform.h" -#include "splinterdb/public_platform.h" #include "splinterdb/default_data_config.h" #include "splinterdb/splinterdb.h" #include "shmem.h" @@ -118,7 +116,7 @@ CTEST2(splinterdb_forked_child, test_data_structures_handles) int rc = splinterdb_create(&splinterdb_cfg, &spl_handle); ASSERT_EQUAL(0, rc); - int pid = platform_getpid(); + int pid = platform_get_os_pid(); // Parent / main() process is always at tid==0. ASSERT_EQUAL(0, platform_get_tid()); @@ -131,6 +129,8 @@ CTEST2(splinterdb_forked_child, test_data_structures_handles) platform_error_log("fork() of child process failed: pid=%d\n", pid); return; } else if (pid == 0) { + platform_register_thread(); + // Verify that child process sees the same handle to a running Splinter // as seen by the parent. (We cross-check using the copy saved off in // shared memory.) @@ -167,17 +167,9 @@ CTEST2(splinterdb_forked_child, test_data_structures_handles) ASSERT_TRUE(platform_valid_addr_in_heap( spl_heap_id, splinterdb_get_memtable_context_handle(spl_handle))); - // Before registering w/Splinter, child process is still at tid==0. - ASSERT_EQUAL(0, platform_get_tid()); - - // Perform some inserts through child process - splinterdb_register_thread(spl_handle); - // After registering w/Splinter, child process' tid will change. ASSERT_EQUAL(1, platform_get_tid()); - splinterdb_deregister_thread(spl_handle); - // After deregistering w/Splinter, child process is back to invalid value ASSERT_EQUAL(INVALID_TID, platform_get_tid()); } @@ -193,6 +185,7 @@ CTEST2(splinterdb_forked_child, test_data_structures_handles) splinterdb_close(&spl_handle); } else { // Child should not attempt to run the rest of the tests + platform_deregister_thread(); exit(0); } } @@ -232,7 +225,7 @@ CTEST2(splinterdb_forked_child, test_one_insert_then_close_bug) int rc = splinterdb_create(&splinterdb_cfg, &spl_handle); ASSERT_EQUAL(0, rc); - int pid = platform_getpid(); + int pid = platform_get_os_pid(); platform_default_log( "Parent OS-pid=%d, ThreadID=%lu\n", pid, platform_get_tid()); @@ -243,11 +236,11 @@ CTEST2(splinterdb_forked_child, test_one_insert_then_close_bug) return; } else if (pid == 0) { // Perform some inserts through child process - splinterdb_register_thread(spl_handle); + platform_register_thread(); platform_default_log("OS-pid=%d, ThreadID=%lu: " "Child execution started ...\n", - platform_getpid(), + platform_get_os_pid(), platform_get_tid()); char key_data[30]; @@ -268,15 +261,13 @@ CTEST2(splinterdb_forked_child, test_one_insert_then_close_bug) key = slice_create(key_len, key_data); rc = splinterdb_insert(spl_handle, key, to_insert); ASSERT_EQUAL(0, rc); - - splinterdb_deregister_thread(spl_handle); } // Only parent can close Splinter if (pid) { platform_default_log("OS-pid=%d, ThreadID=%lu: " "Waiting for child pid=%d to complete ...\n", - platform_getpid(), + platform_get_os_pid(), platform_get_tid(), pid); @@ -285,7 +276,7 @@ CTEST2(splinterdb_forked_child, test_one_insert_then_close_bug) platform_default_log("OS-pid=%d, ThreadID=%lu: " "Child execution wait() completed." " Resuming parent ...\n", - platform_getpid(), + platform_get_os_pid(), platform_get_tid()); // We would get assertions tripping from BTree iterator code here, @@ -293,6 +284,7 @@ CTEST2(splinterdb_forked_child, test_one_insert_then_close_bug) // was not in-place. splinterdb_close(&spl_handle); } else { + platform_deregister_thread(); // child should not attempt to run the rest of the tests exit(0); } @@ -352,7 +344,7 @@ CTEST2(splinterdb_forked_child, int rc = splinterdb_create(&splinterdb_cfg, &spl_handle); ASSERT_EQUAL(0, rc); - int pid = platform_getpid(); + int pid = platform_get_os_pid(); platform_default_log( "Parent OS-pid=%d, ThreadID=%lu\n", pid, platform_get_tid()); @@ -363,26 +355,26 @@ CTEST2(splinterdb_forked_child, return; } else if (pid == 0) { // Perform some inserts through child process - splinterdb_register_thread(spl_handle); + platform_register_thread(); platform_default_log( "OS-pid=%d, ThreadID=%lu: " "Child execution started: Test cache-flush before deregister ...\n", - platform_getpid(), + platform_get_os_pid(), platform_get_tid()); do_many_inserts(spl_handle, data->num_inserts); // This combination of calls tests scenario (1) splinterdb_cache_flush(spl_handle); - splinterdb_deregister_thread(spl_handle); + platform_deregister_thread(); // Repeat the insert exercise: Perform some inserts through child process - splinterdb_register_thread(spl_handle); + platform_register_thread(); platform_default_log( "OS-pid=%d, ThreadID=%lu, Test cache-flush after deregister:\n", - platform_getpid(), + platform_get_os_pid(), platform_get_tid()); do_many_inserts(spl_handle, data->num_inserts); @@ -390,35 +382,33 @@ CTEST2(splinterdb_forked_child, // This combination of calls tests scenario (2) // As part of deregistering a thread, all pending IOs are completed // and the IO-context handle for this thread is reset. - splinterdb_deregister_thread(spl_handle); + platform_deregister_thread(); // **** DEAD CODE WARNING **** // You -cannot- enable this call. A thread is supposed to have // completely drained all its pending IOs, and it cannot do // any more IOs (which is what the flush below will try to do.) // splinterdb_cache_flush(spl_handle); - } - - // Only parent can close Splinter - if (pid) { - platform_default_log("OS-pid=%d, ThreadID=%lu: " - "Waiting for child pid=%d to complete ...\n", - platform_getpid(), - platform_get_tid(), - pid); - - safe_wait(); - platform_default_log("OS-pid=%d, ThreadID=%lu: " - "Child execution wait() completed." - " Resuming parent ...\n", - platform_getpid(), - platform_get_tid()); - splinterdb_close(&spl_handle); - } else { // child should not attempt to run the rest of the tests exit(0); } + + // Only parent gets here + platform_default_log("OS-pid=%d, ThreadID=%lu: " + "Waiting for child pid=%d to complete ...\n", + platform_get_os_pid(), + platform_get_tid(), + pid); + + safe_wait(); + + platform_default_log("OS-pid=%d, ThreadID=%lu: " + "Child execution wait() completed." + " Resuming parent ...\n", + platform_get_os_pid(), + platform_get_tid()); + splinterdb_close(&spl_handle); } /* @@ -458,7 +448,7 @@ CTEST2(splinterdb_forked_child, test_multiple_forked_process_doing_IOs) int rc = splinterdb_create(&splinterdb_cfg, &spl_handle); ASSERT_EQUAL(0, rc); - int pid = platform_getpid(); + int pid = platform_get_os_pid(); platform_default_log( "Parent OS-pid=%d, ThreadID=%lu, fork %lu child processes.\n", @@ -490,13 +480,13 @@ CTEST2(splinterdb_forked_child, test_multiple_forked_process_doing_IOs) } // Perform some inserts through child process - splinterdb_register_thread(spl_handle); + platform_register_thread(); platform_default_log( "OS-pid=%d, ThreadID=%lu:" "Child execution started: Perform %lu (%d million) inserts ..." " Test cache-flush before deregister ...\n", - platform_getpid(), + platform_get_os_pid(), platform_get_tid(), data->num_inserts, (int)(data->num_inserts / MILLION)); @@ -513,7 +503,9 @@ CTEST2(splinterdb_forked_child, test_multiple_forked_process_doing_IOs) * * splinterdb_cache_flush(spl_handle); */ - splinterdb_deregister_thread(spl_handle); + platform_deregister_thread(); + // child should not attempt to run the rest of the tests + exit(0); } } @@ -521,7 +513,7 @@ CTEST2(splinterdb_forked_child, test_multiple_forked_process_doing_IOs) if (data->am_parent && pid) { platform_default_log("OS-pid=%d, ThreadID=%lu: " "Waiting for child pid=%d to complete ...\n", - platform_getpid(), + platform_get_os_pid(), platform_get_tid(), pid); @@ -539,13 +531,10 @@ CTEST2(splinterdb_forked_child, test_multiple_forked_process_doing_IOs) platform_default_log("OS-pid=%d, ThreadID=%lu: " "Child execution wait() completed." " Resuming parent ...\n", - platform_getpid(), + platform_get_os_pid(), platform_get_tid()); splinterdb_close(&spl_handle); - } else { - // child should not attempt to run the rest of the tests - exit(0); } } diff --git a/tests/unit/splinterdb_heap_id_mgmt_test.c b/tests/unit/splinterdb_heap_id_mgmt_test.c index a7bbc44bb..2a8bf0b1e 100644 --- a/tests/unit/splinterdb_heap_id_mgmt_test.c +++ b/tests/unit/splinterdb_heap_id_mgmt_test.c @@ -1,4 +1,4 @@ -// Copyright 2023 VMware, Inc. +// Copyright 2023-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* @@ -19,6 +19,7 @@ * task_system_test.c etc. * ----------------------------------------------------------------------------- */ +#include "platform_units.h" #include "splinterdb/splinterdb.h" #include "splinterdb/default_data_config.h" #include "config.h" @@ -47,6 +48,8 @@ CTEST_SETUP(splinterdb_heap_id_mgmt) { platform_status rc = STATUS_OK; + platform_register_thread(); + bool use_shmem = config_parse_use_shmem(Ctest_argc, (char **)Ctest_argv); uint64 heap_capacity = (256 * MiB); // small heap is sufficient. @@ -71,6 +74,7 @@ CTEST_SETUP(splinterdb_heap_id_mgmt) CTEST_TEARDOWN(splinterdb_heap_id_mgmt) { platform_heap_destroy(&data->kvs_cfg.heap_id); + platform_deregister_thread(); } /* diff --git a/tests/unit/splinterdb_quick_test.c b/tests/unit/splinterdb_quick_test.c index 0071ee43c..f61085806 100644 --- a/tests/unit/splinterdb_quick_test.c +++ b/tests/unit/splinterdb_quick_test.c @@ -1,4 +1,4 @@ -// Copyright 2021 VMware, Inc. +// Copyright 2021-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* @@ -28,9 +28,9 @@ #include #include +#include "platform_threads.h" #include "splinterdb/splinterdb.h" #include "splinterdb/data.h" -#include "splinterdb/public_platform.h" #include "splinterdb/default_data_config.h" #include "unit_tests.h" #include "util.h" @@ -107,15 +107,17 @@ CTEST_DATA(splinterdb_quick) // Optional setup function for suite, called before every test in suite CTEST_SETUP(splinterdb_quick) { + int rc = platform_register_thread(); + ASSERT_EQUAL(0, rc); default_data_config_init(TEST_MAX_KEY_SIZE, &data->default_data_cfg.super); create_default_cfg(&data->cfg, &data->default_data_cfg.super); data->cfg.use_shmem = config_parse_use_shmem(Ctest_argc, (char **)Ctest_argv); - int rc = splinterdb_create(&data->cfg, &data->kvsb); + rc = splinterdb_create(&data->cfg, &data->kvsb); ASSERT_EQUAL(0, rc); ASSERT_TRUE(TEST_MAX_VALUE_SIZE - < MAX_INLINE_MESSAGE_SIZE(LAIO_DEFAULT_PAGE_SIZE)); + < MAX_INLINE_MESSAGE_SIZE(IO_DEFAULT_PAGE_SIZE)); } // Optional teardown function for suite, called after every test in suite @@ -124,6 +126,7 @@ CTEST_TEARDOWN(splinterdb_quick) if (data->kvsb) { splinterdb_close(&data->kvsb); } + platform_deregister_thread(); } /* @@ -272,7 +275,7 @@ CTEST2(splinterdb_quick, test_key_size_gt_max_key_size) CTEST2(splinterdb_quick, test_value_size_gt_max_value_size) { size_t too_large_value_len = - MAX_INLINE_MESSAGE_SIZE(LAIO_DEFAULT_PAGE_SIZE) + 1; + MAX_INLINE_MESSAGE_SIZE(IO_DEFAULT_PAGE_SIZE) + 1; char *too_large_value_data; too_large_value_data = TYPED_ARRAY_MALLOC( data->cfg.heap_id, too_large_value_data, too_large_value_len); diff --git a/tests/unit/splinterdb_stress_test.c b/tests/unit/splinterdb_stress_test.c index 348dc7bfb..bb20f0c15 100644 --- a/tests/unit/splinterdb_stress_test.c +++ b/tests/unit/splinterdb_stress_test.c @@ -1,4 +1,4 @@ -// Copyright 2021 VMware, Inc. +// Copyright 2021-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* @@ -10,11 +10,9 @@ #include #include -#include "splinterdb/public_platform.h" #include "splinterdb/default_data_config.h" #include "splinterdb/splinterdb.h" #include "unit_tests.h" -#include "util.h" #include "../functional/random.h" #include "config.h" #include "ctest.h" // This is required for all test-case files. @@ -23,7 +21,7 @@ #define TEST_VALUE_SIZE 116 // Function Prototypes -static void * +static void exec_worker_thread(void *w); static void @@ -49,6 +47,7 @@ CTEST_DATA(splinterdb_stress) // Setup function for suite, called before every test in suite CTEST_SETUP(splinterdb_stress) { + platform_register_thread(); data->cfg = (splinterdb_config){.filename = TEST_DB_NAME, .cache_size = 1000 * Mega, .disk_size = 9000 * Mega, @@ -69,6 +68,7 @@ CTEST_SETUP(splinterdb_stress) CTEST_TEARDOWN(splinterdb_stress) { splinterdb_close(&data->kvsb); + platform_deregister_thread(); } // Do random inserts across multiple threads @@ -87,8 +87,12 @@ CTEST2(splinterdb_stress, test_random_inserts_concurrent) pthread_t thread_ids[num_threads]; for (int i = 0; i < num_threads; i++) { - int rc = pthread_create(&thread_ids[i], NULL, &exec_worker_thread, &wcfg); - ASSERT_EQUAL(0, rc); + platform_status rc = platform_thread_create(&thread_ids[i], + FALSE, + &exec_worker_thread, + &wcfg, + platform_get_heap_id()); + platform_assert_status_ok(rc); } CTEST_LOG_INFO("Waiting for %d worker threads ...\n", num_threads); @@ -97,13 +101,8 @@ CTEST2(splinterdb_stress, test_random_inserts_concurrent) } for (int i = 0; i < num_threads; i++) { - void *thread_rc; - int rc = pthread_join(thread_ids[i], &thread_rc); - ASSERT_EQUAL(0, rc); - if (thread_rc != 0) { - CTEST_ERR( - "Thread %d [ID=%lu] had error: %p\n", i, thread_ids[i], thread_rc); - } + platform_status rc = platform_thread_join(&thread_ids[i]); + platform_assert_status_ok(rc); } } @@ -151,8 +150,8 @@ CTEST2(splinterdb_stress, test_naive_range_delete) CTEST2(splinterdb_stress, test_iterator_over_many_kvs) { char key_str[KEY_SIZE]; - char *value_str = "This is the value string\0"; - const uint32 inserts = 1 << 25; // 16 million + char value_str[] = "This is the value string\0"; + const uint32 inserts = 1 << 25; // 16 million for (int i = 0; i < inserts; i++) { snprintf(key_str, sizeof(key_str), "key-%08x", i); slice key = slice_create(sizeof(key_str), key_str); @@ -274,7 +273,7 @@ CTEST2_SKIP(splinterdb_stress, test_issue_458_mini_destroy_unused_debug_assert) } // Per-thread workload -static void * +static void exec_worker_thread(void *w) { char key_buf[TEST_KEY_SIZE] = {0}; @@ -285,8 +284,6 @@ exec_worker_thread(void *w) int random_data = wcfg->random_data; splinterdb *kvsb = wcfg->kvsb; - splinterdb_register_thread(kvsb); - pthread_t thread_id = pthread_self(); CTEST_LOG_INFO("Writing lots of data from thread %lu\n", thread_id); @@ -307,9 +304,6 @@ exec_worker_thread(void *w) CTEST_LOG_INFO("Thread %lu has completed %u inserts\n", thread_id, i); } } - - splinterdb_deregister_thread(kvsb); - return 0; } diff --git a/tests/unit/task_system_test.c b/tests/unit/task_system_test.c index b0226435a..441092504 100644 --- a/tests/unit/task_system_test.c +++ b/tests/unit/task_system_test.c @@ -1,4 +1,4 @@ -// Copyright 2022 VMware, Inc. +// Copyright 2022-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* @@ -23,21 +23,18 @@ * programs must only use the external interfaces listed above. * ----------------------------------------------------------------------------- */ -#include "splinterdb/public_platform.h" #include "unit_tests.h" #include "ctest.h" // This is required for all test-case files. #include "platform.h" #include "config.h" // Reqd for definition of master_config{} -#include "core.h" // Needed for trunk_get_scratch_size() #include "task.h" -#include "splinterdb/splinterdb.h" -#include "splinterdb/default_data_config.h" // Configuration for each worker thread typedef struct { task_system *tasks; platform_thread this_thread_id; // OS-generated thread ID threadid exp_thread_idx; // Splinter-generated expected thread index + uint64 *done; } thread_config; // Configuration for worker threads used in lock-step testing exercise @@ -86,8 +83,8 @@ CTEST_DATA(task_system) task_system_config task_cfg; // Following get setup pointing to allocated memory - platform_io_handle *ioh; // Only prerequisite needed to setup task system - task_system *tasks; + io_handle *ioh; // Only prerequisite needed to setup task system + task_system *tasks; }; /* @@ -99,6 +96,8 @@ CTEST_DATA(task_system) */ CTEST_SETUP(task_system) { + platform_register_thread(); + platform_status rc = STATUS_OK; bool use_shmem = config_parse_use_shmem(Ctest_argc, (char **)Ctest_argv); @@ -108,10 +107,6 @@ CTEST_SETUP(task_system) platform_get_module_id(), heap_capacity, use_shmem, &data->hid); platform_assert_status_ok(rc); - // Allocate and initialize the IO sub-system. - data->ioh = TYPED_MALLOC(data->hid, data->ioh); - ASSERT_TRUE((data->ioh != NULL)); - // Do minimal IO config setup, using default IO values. master_config master_cfg; config_set_defaults(&master_cfg); @@ -124,10 +119,8 @@ CTEST_SETUP(task_system) master_cfg.io_async_queue_depth, master_cfg.io_filename); - rc = io_handle_init(data->ioh, &data->io_cfg, data->hid); - ASSERT_TRUE(SUCCESS(rc), - "Failed to init IO handle: %s\n", - platform_status_to_string(rc)); + data->ioh = io_handle_create(&data->io_cfg, data->hid); + ASSERT_TRUE(data->ioh != NULL, "Failed to create IO handle\n"); // Background threads are OFF by default. rc = create_task_system_without_bg_threads(data); @@ -141,8 +134,9 @@ CTEST_SETUP(task_system) CTEST_TEARDOWN(task_system) { task_system_destroy(data->hid, &data->tasks); - io_handle_deinit(data->ioh); + io_handle_destroy(data->ioh); platform_heap_destroy(&data->hid); + platform_deregister_thread(); } /* @@ -206,7 +200,7 @@ CTEST2(task_system, test_one_thread_using_lower_apis) // This main-thread's thread-index remains unchanged. ASSERT_EQUAL(main_thread_idx, platform_get_tid()); - rc = platform_thread_join(new_thread); + rc = platform_thread_join(&new_thread); ASSERT_TRUE(SUCCESS(rc)); // This main-thread's thread-index remains unchanged. @@ -251,13 +245,11 @@ CTEST2(task_system, test_one_thread_using_extern_apis) // This interface packages all the platform_thread_create() and // register / deregister business, around the invocation of the // user's worker function, exec_one_thread_use_extern_apis(). - rc = task_thread_create("test_one_thread", - exec_one_thread_use_extern_apis, - &thread_cfg, - core_get_scratch_size(), - data->tasks, - data->hid, - &new_thread); + rc = platform_thread_create(&new_thread, + FALSE, + exec_one_thread_use_extern_apis, + &thread_cfg, + data->hid); ASSERT_TRUE(SUCCESS(rc)); thread_cfg.this_thread_id = new_thread; @@ -267,7 +259,7 @@ CTEST2(task_system, test_one_thread_using_extern_apis) // task_thread_join(), if it were to exist, would have been // a pass-through to platform-specific join() method, anyways. - rc = platform_thread_join(new_thread); + rc = platform_thread_join(&new_thread); ASSERT_TRUE(SUCCESS(rc)); // This main-thread's thread-index remains unchanged. @@ -295,23 +287,25 @@ CTEST2(task_system, test_max_threads_using_lower_apis) platform_status rc = STATUS_OK; ZERO_ARRAY(thread_cfg); - ASSERT_EQUAL(task_get_max_tid(data->tasks), + ASSERT_EQUAL(platform_num_threads(), 1, - "Before threads start, task_get_max_tid() = %lu", - task_get_max_tid(data->tasks)); + "Before threads start, platform_num_threads() = %lu", + platform_num_threads()); // We may have started some background threads, if this test was so // configured. So, start-up all the remaining threads. - threadid max_tid_so_far = task_get_max_tid(data->tasks); + threadid max_tid_so_far = platform_num_threads(); // Start-up n-threads, record their expected thread-IDs, which will be // validated by the thread's execution function below. + uint64 done = 0; for (tctr = max_tid_so_far; tctr < ARRAY_SIZE(thread_cfg); tctr++) { thread_cfgp = &thread_cfg[tctr]; // These are independent of the new thread's creation. thread_cfgp->tasks = data->tasks; thread_cfgp->exp_thread_idx = tctr; + thread_cfgp->done = &done; rc = platform_thread_create( &new_thread, FALSE, exec_one_of_n_threads, thread_cfgp, data->hid); @@ -323,7 +317,7 @@ CTEST2(task_system, test_max_threads_using_lower_apis) // Complete execution of n-threads. Worker fn does the validation. for (tctr = max_tid_so_far; tctr < ARRAY_SIZE(thread_cfg); tctr++) { thread_cfgp = &thread_cfg[tctr]; - rc = platform_thread_join(thread_cfgp->this_thread_id); + rc = platform_thread_join(&thread_cfgp->this_thread_id); ASSERT_TRUE(SUCCESS(rc)); } } @@ -366,20 +360,18 @@ CTEST2(task_system, test_use_all_but_one_threads_for_bg_threads) thread_config_lockstep thread_cfg[2]; ZERO_ARRAY(thread_cfg); thread_cfg[0].tasks = data->tasks; - thread_cfg[0].exp_thread_idx = task_get_max_tid(data->tasks); + thread_cfg[0].exp_thread_idx = platform_num_threads(); thread_cfg[0].exp_max_tid = MAX_THREADS; thread_cfg[0].line = __LINE__; platform_thread new_thread[2] = {0}; // This should successfully create a new (the last) thread - rc = task_thread_create("test_one_thread", - exec_user_thread_loop_for_stop, - &thread_cfg[0], - core_get_scratch_size(), - data->tasks, - data->hid, - &new_thread[0]); + rc = platform_thread_create(&new_thread[0], + FALSE, + exec_user_thread_loop_for_stop, + &thread_cfg[0], + data->hid); ASSERT_TRUE(SUCCESS(rc)); // Wait till 1st user-thread gets to its wait-for-stop loop @@ -387,28 +379,26 @@ CTEST2(task_system, test_use_all_but_one_threads_for_bg_threads) platform_sleep_ns(USEC_TO_NSEC(100000)); // 100 msec. } thread_cfg[1].tasks = data->tasks; - thread_cfg[1].exp_thread_idx = task_get_max_tid(data->tasks); + thread_cfg[1].exp_thread_idx = platform_num_threads(); // We've used up all threads. This thread creation should fail. - rc = task_thread_create("test_one_thread", - exec_user_thread_loop_for_stop, - &thread_cfg[1], - core_get_scratch_size(), - data->tasks, - data->hid, - &new_thread[1]); + rc = platform_thread_create(&new_thread[1], + FALSE, + exec_user_thread_loop_for_stop, + &thread_cfg[1], + data->hid); ASSERT_FALSE(SUCCESS(rc), "Thread should not have been created" - ", new_thread=%lu, max_tid=%lu\n", + ", new_thread=%lu, num_threads=%lu\n", new_thread[1], - task_get_max_tid(data->tasks)); + platform_num_threads()); // Stop the running user-thread now that our test is done. thread_cfg[0].stop_thread = TRUE; for (uint64 tctr = 0; tctr < ARRAY_SIZE(new_thread); tctr++) { if (new_thread[tctr]) { - rc = platform_thread_join(new_thread[tctr]); + rc = platform_thread_join(&new_thread[tctr]); ASSERT_TRUE(SUCCESS(rc)); } } @@ -429,10 +419,9 @@ create_task_system_without_bg_threads(void *datap) uint64 num_bg_threads[NUM_TASK_TYPES] = {0}; rc = task_system_config_init(&data->task_cfg, TRUE, // use stats - num_bg_threads, - core_get_scratch_size()); + num_bg_threads); ASSERT_TRUE(SUCCESS(rc)); - rc = task_system_create(data->hid, data->ioh, &data->tasks, &data->task_cfg); + rc = task_system_create(data->hid, &data->tasks, &data->task_cfg); return rc; } @@ -456,21 +445,20 @@ create_task_system_with_bg_threads(void *datap, num_bg_threads[TASK_TYPE_NORMAL] = num_normal_bg_threads; rc = task_system_config_init(&data->task_cfg, TRUE, // use stats - num_bg_threads, - core_get_scratch_size()); + num_bg_threads); ASSERT_TRUE(SUCCESS(rc)); - rc = task_system_create(data->hid, data->ioh, &data->tasks, &data->task_cfg); + rc = task_system_create(data->hid, &data->tasks, &data->task_cfg); if (!SUCCESS(rc)) { return rc; } // Wait-for all background threads to startup. uint64 nbg_threads = (num_memtable_bg_threads + num_normal_bg_threads); - threadid max_thread_id = task_get_max_tid(data->tasks); + threadid max_thread_id = platform_num_threads(); while (max_thread_id < nbg_threads) { platform_sleep_ns(USEC_TO_NSEC(100000)); // 100 msec. - max_thread_id = task_get_max_tid(data->tasks); + max_thread_id = platform_num_threads(); } return rc; } @@ -489,47 +477,12 @@ exec_one_thread_use_lower_apis(void *arg) { thread_config *thread_cfg = (thread_config *)arg; - // This is the important call to initialize thread-specific stuff in - // Splinter's task-system, which sets up the thread-id (index) and records - // this thread as active with the task system. - task_register_this_thread(thread_cfg->tasks, core_get_scratch_size()); - threadid this_threads_idx = platform_get_tid(); ASSERT_EQUAL(thread_cfg->exp_thread_idx, this_threads_idx, "exp_thread_idx=%lu, this_threads_idx=%lu\n", thread_cfg->exp_thread_idx, this_threads_idx); - - // Registration should have allocated some scratch space memory. - ASSERT_TRUE( - core_get_scratch_size() == 0 - || task_system_get_thread_scratch(thread_cfg->tasks, platform_get_tid()) - != NULL); - - // Brain-dead cross-check, to understand what's going on with thread-IDs. - platform_thread thread_id = platform_thread_id_self(); - ASSERT_TRUE((thread_cfg->this_thread_id == thread_id) - || (thread_cfg->this_thread_id == 0)); - - task_deregister_this_thread(thread_cfg->tasks); - - // Deregistration releases scratch space memory. - ASSERT_TRUE( - core_get_scratch_size() == 0 - || task_system_get_thread_scratch(thread_cfg->tasks, this_threads_idx) - == NULL); - - // Register / de-register of thread with SplinterDB's task system is - // SplinterDB's jugglery to keep track of resources. get_tid() should - // now be reset. - threadid get_tid_after_deregister = platform_get_tid(); - ASSERT_EQUAL(INVALID_TID, - get_tid_after_deregister, - "get_tid_after_deregister=%lu is != expected index into" - " thread array, %lu ", - get_tid_after_deregister, - thread_cfg->exp_thread_idx); } /* @@ -554,13 +507,7 @@ exec_one_thread_use_extern_apis(void *arg) * The interface used here has already registered this thread. An attempt to * re-register this thread will trip an assertion. (Left here for posterity.) */ - // task_register_this_thread(thread_cfg->tasks, trunk_get_scratch_size()); - - // Registration should have allocated some scratch space memory. - ASSERT_TRUE( - core_get_scratch_size() == 0 - || task_system_get_thread_scratch(thread_cfg->tasks, this_threads_idx) - != NULL); + // task_register_this_thread(thread_cfg->tasks); /* * Dead Code Warning! @@ -586,15 +533,6 @@ exec_one_of_n_threads(void *arg) { thread_config *thread_cfg = (thread_config *)arg; - // Before registration, thread ID should be in an uninit'ed state - ASSERT_EQUAL(INVALID_TID, platform_get_tid()); - - platform_status rc = - task_register_this_thread(thread_cfg->tasks, core_get_scratch_size()); - ASSERT_TRUE(SUCCESS(rc), - "task_register_this_thread() failed: thread idx is %lu", - platform_get_tid()); - threadid this_threads_index = platform_get_tid(); ASSERT_TRUE((this_threads_index < MAX_THREADS), @@ -602,23 +540,16 @@ exec_one_of_n_threads(void *arg) thread_cfg->exp_thread_idx, this_threads_index); + if (platform_num_threads() == MAX_THREADS) { + *thread_cfg->done = 1; + return; + } + // Test case is carefully constructed to fire-up n-threads. Wait for // them to all start-up. - while (task_get_max_tid(thread_cfg->tasks) < MAX_THREADS) { + while (*thread_cfg->done == 0) { platform_sleep_ns(USEC_TO_NSEC(100000)); // 100 msec. } - - task_deregister_this_thread(thread_cfg->tasks); - - // Register / de-register of thread with SplinterDB's task system is just - // SplinterDB's jugglery to keep track of resources. Deregistration should - // have re-init'ed the thread ID. - threadid get_tid_after_deregister = platform_get_tid(); - ASSERT_EQUAL(INVALID_TID, - get_tid_after_deregister, - "get_tid_after_deregister=%lu should be an invalid tid, %lu", - get_tid_after_deregister, - INVALID_TID); } /* @@ -637,8 +568,8 @@ exec_user_thread_loop_for_stop(void *arg) // We should have used up all available threads. Next create should fail. ASSERT_EQUAL(thread_cfg->exp_max_tid, - task_get_max_tid(thread_cfg->tasks), - "Max tid is incorrect for thread created on line=%d\n", + platform_num_threads(), + "Num threads is incorrect for thread created on line=%d\n", thread_cfg->line); // The calling interface has already registered this thread. All we do @@ -651,4 +582,4 @@ exec_user_thread_loop_for_stop(void *arg) CTEST_LOG_INFO("Last user thread ID=%lu, created on line=%d exiting ...\n", this_threads_idx, thread_cfg->line); -} +} \ No newline at end of file diff --git a/tests/unit/unit_test_template_dot_c b/tests/unit/unit_test_template_dot_c index 8e8b4e4bf..cfa5b8022 100644 --- a/tests/unit/unit_test_template_dot_c +++ b/tests/unit/unit_test_template_dot_c @@ -1,4 +1,4 @@ -// Copyright 2021 VMware, Inc. +// Copyright 2021-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* diff --git a/tests/unit/unit_tests.h b/tests/unit/unit_tests.h index 319618e0f..017959948 100644 --- a/tests/unit/unit_tests.h +++ b/tests/unit/unit_tests.h @@ -1,11 +1,10 @@ -// Copyright 2021 VMware, Inc. +// Copyright 2021-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* * unit_tests.h: SplinterDB Unit tests common header file. * Define things in here that are going to be needed across most unit tests. */ -#include "splinterdb/public_platform.h" #include "ctest.h" /* Name of SplinterDB device created for unit-tests */ diff --git a/tests/unit/unit_tests_common.c b/tests/unit/unit_tests_common.c index 298c83db7..42ae54e72 100644 --- a/tests/unit/unit_tests_common.c +++ b/tests/unit/unit_tests_common.c @@ -1,4 +1,4 @@ -// Copyright 2023 VMware, Inc. +// Copyright 2023-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* @@ -10,6 +10,7 @@ */ #include "ctest.h" // This is required for all test-case files. #include "unit_tests.h" +#include "splinterdb/public_platform.h" /* * Setup function is provided to manage output log streams from (unit) tests. diff --git a/tests/unit/util_test.c b/tests/unit/util_test.c index 95927f18e..46409339e 100644 --- a/tests/unit/util_test.c +++ b/tests/unit/util_test.c @@ -1,4 +1,4 @@ -// Copyright 2021 VMware, Inc. +// Copyright 2021-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* @@ -25,10 +25,16 @@ check_one_debug_hex_encode(size_t dst_len, CTEST_DATA(util){}; // Optional setup function for suite, called before every test in suite -CTEST_SETUP(util) {} +CTEST_SETUP(util) +{ + platform_register_thread(); +} // Optional teardown function for suite, called after every test in suite -CTEST_TEARDOWN(util) {} +CTEST_TEARDOWN(util) +{ + platform_deregister_thread(); +} /* * Test debug_hex_encode() in all its variations. diff --git a/tests/unit/vector_test.c b/tests/unit/vector_test.c index 0bd42badd..d3ad2ebd9 100644 --- a/tests/unit/vector_test.c +++ b/tests/unit/vector_test.c @@ -1,4 +1,4 @@ -// Copyright 2021 VMware, Inc. +// Copyright 2021-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* @@ -23,6 +23,7 @@ CTEST_DATA(vector) // Optional setup function for suite, called before every test in suite CTEST_SETUP(vector) { + platform_register_thread(); platform_heap_id hid = platform_get_heap_id(); vector_init(&data->empty, hid); vector_init(&data->one, hid); @@ -41,6 +42,7 @@ CTEST_TEARDOWN(vector) vector_deinit(&data->empty); vector_deinit(&data->one); vector_deinit(&data->ten); + platform_deregister_thread(); } CTEST2(vector, length) diff --git a/tests/unit/writable_buffer_test.c b/tests/unit/writable_buffer_test.c index 3380aab16..031379cbe 100644 --- a/tests/unit/writable_buffer_test.c +++ b/tests/unit/writable_buffer_test.c @@ -1,4 +1,4 @@ -// Copyright 2021 VMware, Inc. +// Copyright 2021-2026 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 /* @@ -8,8 +8,7 @@ * Exercises the Writable Buffer interfaces. * ----------------------------------------------------------------------------- */ -#include "splinterdb/public_platform.h" -#include "platform.h" +#include "platform_units.h" #include "config.h" #include "unit_tests.h" #include "ctest.h" // This is required for all test-case files. @@ -31,6 +30,7 @@ CTEST_DATA(writable_buffer) // Optional setup function for suite, called before every test in suite CTEST_SETUP(writable_buffer) { + platform_register_thread(); data->use_shmem = config_parse_use_shmem(Ctest_argc, (char **)Ctest_argv); platform_status rc = platform_heap_create( @@ -42,6 +42,7 @@ CTEST_SETUP(writable_buffer) CTEST_TEARDOWN(writable_buffer) { platform_heap_destroy(&data->hid); + platform_deregister_thread(); } /*