Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 37 additions & 10 deletions .github/workflows/codspeed.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,17 +10,23 @@ on:
permissions:
contents: read

env:
UV_FROZEN: true
UV_PYTHON: 3.13 # use the latest version of Python because it is faster

jobs:
benchmarks:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "3.13" # use the latest version of Python because it is faster
python-version: ${{ env.UV_PYTHON }}

- name: Install python dependencies.
run: pip install -r pytests/requirements.txt
- name: Install UV.
uses: astral-sh/setup-uv@v6
with:
enable-cache: true

- name: Install rust.
id: rust-toolchain
Expand All @@ -31,24 +37,45 @@ jobs:
- name: Cache rust.
uses: Swatinem/rust-cache@v2

- name: Compile.
run: pip install -v -e .
- name: Build wheel with profile generation.
uses: PyO3/maturin-action@v1
with:
manylinux: auto
args: --release --out pgo-wheel --interpreter ${{ env.UV_PYTHON }}
rust-toolchain: 1.76.0
docker-options: -e CI
env:
RUSTFLAGS: "-Cprofile-generate=${{ github.workspace }}/profdata"

- name: Gather PGO data.
run: pytest . --benchmark-enable
run: |
uv sync --group testing
uv pip install libipld --no-index --no-deps --find-links pgo-wheel --force-reinstall

uv run pytest . --benchmark-enable

- name: Prepare merged PGO data.
run: rustup run 1.76.0 bash -c '$RUSTUP_HOME/toolchains/$RUSTUP_TOOLCHAIN/lib/rustlib/x86_64-unknown-linux-gnu/bin/llvm-profdata merge -o ${{ github.workspace }}/merged.profdata ${{ github.workspace }}/profdata'

- name: Compile with profile.
run: pip install -v -e .
- name: Build PGO-optimized wheel.
uses: PyO3/maturin-action@v1
with:
manylinux: auto
args: --release --out dist --interpreter ${{ env.UV_PYTHON }}
rust-toolchain: 1.76.0
docker-options: -e CI
env:
RUSTFLAGS: "-Cprofile-use=${{ github.workspace }}/merged.profdata"

- name: Run benchmarks.
- name: Find PGO built wheel.
id: pgo-wheel
run: echo "path=$(ls dist/*.whl)" | tee -a "$GITHUB_OUTPUT"

- name: Install PGO wheel.
run: uv pip install ${{ steps.pgo-wheel.outputs.path }} --force-reinstall

- name: Run CodSpeed benchmarks.
uses: CodSpeedHQ/action@v3
with:
token: ${{ secrets.CODSPEED_TOKEN }}
run: pytest . --codspeed -n auto
run: uv run --group=codspeed pytest . --codspeed -n auto
16 changes: 10 additions & 6 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ on:
push:
branches:
- main
pull_request:
tags:
- 'v*.*.*'
workflow_dispatch:
Expand Down Expand Up @@ -124,8 +125,12 @@ jobs:
run: |
git config --global core.autocrlf false
git config --global core.eol lf
- uses: actions/checkout@v4
- uses: actions/setup-python@v5

- name: Checkout repository.
uses: actions/checkout@v4

- name: Install UV.
uses: astral-sh/setup-uv@v6
with:
python-version: ${{ matrix.interpreter }}

Expand All @@ -151,11 +156,10 @@ jobs:

- name: Gather PGO data.
run: |
pip install -U pip
pip install -r pytests/requirements.txt
uv sync --group testing
uv pip install libipld --no-index --no-deps --find-links pgo-wheel --force-reinstall

pip install libipld --no-index --no-deps --find-links pgo-wheel --force-reinstall
pytest . --benchmark-enable
uv run pytest . --benchmark-enable

# we can't use github.workspace here because of Windows with backslashes
rustup run 1.76.0 bash -c 'echo LLVM_PROFDATA=$RUSTUP_HOME/toolchains/$RUSTUP_TOOLCHAIN/lib/rustlib/${{ env.RUST_HOST }}/bin/llvm-profdata >> "$GITHUB_ENV"'
Expand Down
21 changes: 12 additions & 9 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,9 @@ on: [ pull_request ]
permissions:
contents: read

env:
UV_PYTHON: 3.8 # minimum supported version

jobs:
unit_tests:
runs-on: ubuntu-latest
Expand All @@ -13,16 +16,16 @@ jobs:
- name: Checkout repository.
uses: actions/checkout@v4

- name: Set up Python.
uses: actions/setup-python@v5
with:
# minimum supported version
python-version: 3.8
- name: Install UV.
uses: astral-sh/setup-uv@v6

- name: Install dependencies.
run: |
pip install -r pytests/requirements.txt
pip install -v -e .
run: uv sync --group testing

- name: Compile.
run: uv pip install -v -e .
env:
RUST_BACKTRACE: 1

- name: Run Tests.
run: pytest
run: uv run pytest
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -28,3 +28,6 @@ test.py
__pycache__
.benchmarks
.pytest_cache

# Build artifacts
*.so
1 change: 0 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,6 @@ print(libipld.encode_multibase('u', b'yes mani !'))
- Encode Multibase (`encode_multibase(str, bytes) -> str`). Accepts base and data.
- Decode CAR (`decode_car(bytes) -> tuple[dict, dict[bytes, dict]]`). Returns a header and blocks mapped by CID. CIDs in raw byte form.

Note: a stub file will be provided in the future.

## Requirements

Expand Down
21 changes: 21 additions & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -37,13 +37,34 @@ classifiers = [
"Tracker" = "https://github.com/MarshalX/python-libipld/issues"
"Author" = "https://github.com/MarshalX"

[dependency-groups]
dev = ["maturin>=1.2,<2.0"]
testing = [
{ include-group = "dev" },
"pytest==8.3.5",
"pytest-benchmark==4.0.0",
"pytest-xdist==3.6.1",
]
codspeed = [
# only run on CI with the latest Python version
'pytest-codspeed==3.2.0; python_version == "3.13" and implementation_name == "cpython"',
]

all = [
{ include-group = "dev" },
{ include-group = "testing" },
]

[tool.pytest.ini_options]
addopts = [
'--benchmark-columns', 'min,mean,stddev,outliers,rounds,iterations',
'--benchmark-disable', # use --benchmark-enable
]

[tool.maturin]
python-source = "python"
module-name = "libipld._libipld"
bindings = "pyo3"
features = ["pyo3/extension-module"]

[build-system]
Expand Down
4 changes: 0 additions & 4 deletions pytests/requirements.txt

This file was deleted.

21 changes: 21 additions & 0 deletions python/libipld/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
from ._libipld import (
decode_cid,
encode_cid,
encode_dag_cbor,
decode_car,
decode_dag_cbor,
decode_dag_cbor_multi,
decode_multibase,
encode_multibase,
)

__all__ = [
"decode_cid",
"encode_cid",
"encode_dag_cbor",
"decode_car",
"decode_dag_cbor",
"decode_dag_cbor_multi",
"decode_multibase",
"encode_multibase",
]
101 changes: 101 additions & 0 deletions python/libipld/_libipld.pyi
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
from __future__ import annotations
from typing import Any


def decode_cid(data: str | bytes) -> dict[str, Any]:
"""Decode a CID from either its string representation or raw bytes.

Args:
data: Either a CID string (e.g. 'bafy...') or raw CID bytes

Returns:
A dict containing:
- version: int (0 or 1)
- codec: int (e.g. 113 for DAG-CBOR)
- hash: dict containing:
- code: int (hash algorithm code)
- size: int (hash size in bytes)
- digest: bytes (hash digest)
"""


def encode_cid(data: str | bytes) -> str:
"""Encode a CID to its string representation.

Args:
data: Either a CID string (will be returned as-is) or raw CID bytes

Returns:
A CID string (e.g. 'bafy...')
"""


def decode_car(data: bytes) -> tuple[dict[str, Any], dict[bytes, dict[str, Any]]]:
"""Decode a CAR file.

Args:
data: Raw CAR file bytes

Returns:
A tuple containing:
- header: dict (CAR header)
- blocks: dict mapping CID bytes to block data
"""


def decode_dag_cbor(data: bytes) -> Any:
"""Decode DAG-CBOR data to Python objects.

Args:
data: Raw DAG-CBOR bytes

Returns:
A Python object
"""


def decode_dag_cbor_multi(data: bytes) -> list[Any]:
"""Decode multiple DAG-CBOR objects from bytes.

Args:
data: Raw DAG-CBOR bytes containing multiple objects

Returns:
A list of Python objects
"""


def encode_dag_cbor(data: Any) -> bytes:
"""Encode Python objects to DAG-CBOR.

Args:
data: Any Python object that can be encoded to DAG-CBOR

Returns:
Raw DAG-CBOR bytes
"""


def decode_multibase(data: str) -> tuple[str, bytes]:
"""Decode multibase-encoded data.

Args:
data: Multibase-encoded string (e.g. 'ueWVzIG1hbmkgIQ')

Returns:
A tuple containing:
- base: str (the base code, e.g. 'u')
- data: bytes (the decoded data)
"""


def encode_multibase(code: str, data: bytes | str) -> str:
"""Encode data using multibase.

Args:
code: Base code (e.g. 'u' for base58btc)
data: Data to encode (bytes or string that can be converted to bytes)

Returns:
Multibase-encoded string
"""
1 change: 1 addition & 0 deletions src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -577,6 +577,7 @@ fn get_err(msg: &str, err: String) -> PyErr {
}

#[pymodule]
#[pyo3(name = "_libipld")]
fn libipld(m: &Bound<'_, PyModule>) -> PyResult<()> {
m.add_function(wrap_pyfunction!(decode_cid, m)?)?;
m.add_function(wrap_pyfunction!(encode_cid, m)?)?;
Expand Down
Loading