Distribute rules via http_archive

Author: bcmyers

.bazelignore

@@ -1 +1,2 @@
+examples
 node_modules

.github/workflows/main.yaml

@@ -4,11 +4,15 @@ on:
     branches: [main]
   pull_request:
   workflow_dispatch:
+env:
+  CI: 1
 jobs:
   ci:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - uses: bazelbuild/setup-bazelisk@v1
-      - run: bazel test --config=ci //...
-      - run: bazel run //:gazelle -- -mode diff || exit 1
+      - uses: extractions/setup-just@6e1de3cc407de738551abd6c0923bd5ed5608042
+        with:
+          just-version: "1.38.0"
+      - run: just test

BUILD.bazel

@@ -1,4 +1,4 @@
-load("@gazelle//:def.bzl", "DEFAULT_LANGUAGES", "gazelle", "gazelle_binary")
+load("@bazel_gazelle//:def.bzl", "DEFAULT_LANGUAGES", "gazelle", "gazelle_binary")
 load("@npm//:defs.bzl", "npm_link_all_packages")
 load("@rules_pkg//pkg:pkg.bzl", "pkg_tar")
 
@@ -20,6 +20,17 @@ gazelle(
     gazelle = ":gazelle-binary",
 )
 
+gazelle(
+    name = "gazelle-update-repos",
+    args = [
+        "-build_file_proto_mode=disable",
+        "-from_file=go.mod",
+        "-prune=true",
+        "-to_macro=oci/private/repositories/go_repositories.bzl%go_repositories",
+    ],
+    command = "update-repos",
+)
+
 gazelle_binary(
     name = "gazelle-binary",
     languages = DEFAULT_LANGUAGES + [
@@ -38,6 +49,7 @@ pkg_tar(
         "//oci:files",
         "//oci/private:files",
         "//oci/private/repositories:files",
+        "//oci/repositories:files",
     ],
     empty_files = ["BUILD.bazel"],
     extension = "tar.gz",

MODULE.bazel

@@ -3,27 +3,55 @@ module(
     repo_name = "com_github_datadog_rules_oci",
 )
 
+# 2024-05-08
 bazel_dep(name = "aspect_bazel_lib", version = "2.7.3")
+
+# 2024-12-04
 bazel_dep(name = "aspect_rules_js", version = "2.1.2")
+
+# 2024-12
 bazel_dep(name = "aspect_rules_lint", version = "1.0.8")
+
+# 2024-04-25
 bazel_dep(name = "bazel_skylib", version = "1.6.1")
-bazel_dep(name = "gazelle", version = "0.38.0")
+
+# 2024-08-01
+bazel_dep(name = "gazelle", version = "0.38.0", repo_name = "bazel_gazelle")
+
+# 2024-05-06
 bazel_dep(name = "rules_go", version = "0.47.1", repo_name = "io_bazel_rules_go")
+
+# 2024-11-06
 bazel_dep(name = "rules_nodejs", version = "6.3.2")
+
+# 2024-02-08
 bazel_dep(name = "rules_pkg", version = "0.10.1")
+
+# 2023-08-11
 bazel_dep(name = "stardoc", version = "0.6.2")
 
+# 2024-04-25
 bazel_dep(name = "bazel_skylib_gazelle_plugin", version = "1.6.1", dev_dependency = True)
+
+# 2024-08-27
 bazel_dep(name = "buildifier_prebuilt", version = "7.3.1", dev_dependency = True)
 
+single_version_override(
+    module_name = "bazel_skylib_gazelle_plugin",
+    patch_strip = 2,
+    patches = [
+        "//third_party/com_github_bazelbuild_bazel-skylib:01-pr-535-support-resolve.patch",
+    ],
+)
+
 go_sdk = use_extension("@io_bazel_rules_go//go:extensions.bzl", "go_sdk")
 go_sdk.download(
     name = "go_sdk",
     version = "1.22.5",
 )
 use_repo(go_sdk, "go_sdk")
 
-go_deps = use_extension("@gazelle//:extensions.bzl", "go_deps")
+go_deps = use_extension("@bazel_gazelle//:extensions.bzl", "go_deps")
 go_deps.from_file(go_mod = "//:go.mod")
 use_repo(
     go_deps,
@@ -36,7 +64,6 @@ use_repo(
     "com_github_sirupsen_logrus",
     "com_github_stretchr_testify",
     "com_github_urfave_cli_v2",
-    "land_oras_oras_go",
 )
 go_deps.module_override(
     patch_strip = 1,
@@ -46,7 +73,7 @@ go_deps.module_override(
     path = "github.com/containerd/containerd",
 )
 
-oci_pull = use_repo_rule("//oci:repositories.bzl", "oci_pull")
+oci_pull = use_repo_rule("//oci:defs.bzl", "oci_pull")
 
 oci_pull(
     name = "ubuntu_noble",

MODULE.bazel.lock

@@ -1,83 +1,58 @@
-## `rules_oci` - blazing fast Bazel rules for building OCI Images
+## `rules_oci` - Bazel rules for building OCI Images
 
 RULES_OCI IS HIGHLY EXPERIMENTAL WITH PLANNED BREAKING CHANGES, PLEASE DO NOT
 DEPEND ON FOR PRODUCTION USE-CASES.
 
 A Bazel rule-set for extending, creating and publishing OCI artifacts, including image
-manifests, image indexes (multi-arch images) and custom artifacts
-([ORAS](https://github.com/oras-project)), with a focus on:
+manifests and image indexes (multi-arch images), with a focus on:
 
-- **Speed**, only pulling artifacts that are needed at build-time (no more long image pull times)
-- **Extensibility**, creating custom artifacts to leverage standard OCI distribution
-  APIs
+- **Extensibility**, creating custom artifacts to leverage standard OCI distribution APIs
 - **Multi-arch images**, compiling and building multi-arch images with a single Bazel invocation
 
-In addition to Bazel rules, we offer many helpers for interacting with OCI
-artifacts under the `go/pkg` directory and a CLI tool for creating new OCI
-artifacts.
-
-`rules_oci` makes an effort to support Docker media types, but there is no
-guarantee of long-term support. Most CRI support the OCI types or there are
-tools available to convert [between the
-specifications](https://github.com/opencontainers/image-spec/blob/v1.0.2/conversion.md).
-
 ### Setup
 
-```
-# Load OCI Bootstrapping rules or copy the rule into your repository.
-git_repository(
-    name = "rules_oci_bootstrap",
-    remote = "https://github.com/DataDog/rules_oci_bootstrap.git",
-    commit = "75330296a80c4a5bfa228dc585ca9a9c3e56d45d",
-)
+```starlark
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 
-load("@rules_oci_bootstrap//:defs.bzl", "oci_blob_pull")
-oci_blob_pull(
+http_archive(
     name = "com_github_datadog_rules_oci",
-    digest = "sha256:cc6c59ed7da6bb376552461e06068f883bbe335359c122c15dce3c24e19cd8e2",
-    extract = True,
-    registry = "ghcr.io",
-    repository = "datadog/rules_oci/rules",
-    type = "tar.gz",
+    sha256 = "<SHA256>",
+    strip_prefix = "rules_oci-<COMMIT>",
+    url = "https://github.com/DataDog/rules_oci/archive/<COMMIT>.tar.gz",
 )
-```
 
-### Docs
+load(
+    "@com_github_datadog_rules_oci//oci/repositories:01_direct_dependencies.bzl",
+    "rules_oci_direct_dependencies",
+)
 
-[Rule API](docs/docs.md)
+rules_oci_direct_dependencies()
 
-Examples can be found in the `tests` directory.
+load("@com_github_datadog_rules_oci//oci/repositories:02_toolchains.bzl", "rules_oci_toolchains")
 
-### How it works at a high level
+rules_oci_toolchains(
+    # Only set this if you have not already registered a go toolchain
+    register_go_toolchain_version = "1.22.5",
 
-At fetch-time we only pull down the manifest json that represents the
-structure of the image, rather than pull down everything -- we call this a shallow
-pull. We then modify the manifest and republish it with just the changed layers
-at "bazel run"-time.
+    # Only set this if you have not already registered a rust toolchain
+    register_rust_toolchain_version = "1.82.0",
+)
 
-This is perfect for the use-case of creating "application images", aka images
-where you just plop a binary on top of a base image. Some additional small
-changes can be done such as injecting a shared library or a config file.
+load(
+    "@com_github_datadog_rules_oci//oci/repositories:03_third_party_go_and_rust_libraries.bzl",
+    "rules_oci_third_party_go_and_rust_libraries",
+)
 
-We've found in most cases we don't need to pull these additional layers as they
-were pushed there previously or can copy (via the mount api) within the same
-registry.
+rules_oci_third_party_go_and_rust_libraries()
+```
 
-This has the downside that there is no verification of all of the content
-in the image, but this trade-off is worth the speed of not downloaded many GBs of
-base images.
+### Docs
 
-### Roadmap
+- [rules](docs/defs.md)
+- [providers](docs/providers.md)
+- [pull repository rule](docs/pull.md)
 
-- [ ] Flesh out code for non-shallow pulls and cases where the layers are coming
-      from a different registry.
-- [ ] Full Starlark DSL for creating custom artifacts, it's currently looks
-      a bit wonky
-- [ ] Support for the ORAS Artifact Spec
-- [ ] Support for custom artifact crawlers to pull artifacts that have children
-      not represented by the OCI Image Spec. Ex pulling a full CNAB bundle and all
-      dependencies.
-- [ ] Benchmark against `rules_docker` and raw `docker build`.
+Examples can be found in the `examples` directory.
 
 ### FAQ
 
@@ -94,12 +69,12 @@ base images.
 
 ### Developing
 
-| action                   | command                               |
-| ------------------------ | ------------------------------------- |
-| Run the tests            | `just test`                           |
-| Run the formatter        | `just format`                         |
-| Run gazelle              | `just gazelle`                        |
-| Update the docs          | `just update-docs`                    |
-| Update go dependencies   | `bazel run //:go -- get <DEPENDENCY>` |
-| Update rust dependencies | `just update-crates`                  |
-| Publish a new release    | `just release`                        |
+| action                   | command                                                  |
+| ------------------------ | -------------------------------------------------------- |
+| Run the tests            | `just test`                                              |
+| Run the formatter        | `just format`                                            |
+| Run gazelle              | `just gazelle`                                           |
+| Update the docs          | `just update-docs`                                       |
+| Update go dependencies   | Modify `go.mod` and run `just update-go-3rd-party`       |
+| Update rust dependencies | Modify `Cargo.toml` and run `just update-rust-3rd-party` |
+| Publish a new release    | `just release`                                           |

README.md

@@ -29,7 +29,7 @@ rust_register_toolchains(
 load("@rules_rust//crate_universe:defs.bzl", "crates_repository")
 
 crates_repository(
-    name = "crate_index",
+    name = "com_github_datadog_rules_oci_crate_index",
     cargo_lockfile = "//:Cargo.lock",
     lockfile = "//:cargo-bazel-lock.json",
     manifests = [
@@ -39,6 +39,6 @@ crates_repository(
     rust_version = _RUSTC_VERSION,
 )
 
-load("@crate_index//:defs.bzl", "crate_repositories")
+load("@com_github_datadog_rules_oci_crate_index//:defs.bzl", "crate_repositories")
 
 crate_repositories()

WORKSPACE

@@ -1,5 +1,5 @@
 {
-  "checksum": "030e153464f00aa0a15a3162aa3e206b5e42faf52412bf783ec58f62406c22bc",
+  "checksum": "bcebbb48f8aa8d2b6ef3a9c5b2cb5c8698f2f2eaa50fdd72a9e863b4b3314d6e",
   "crates": {
     "aho-corasick 1.1.3": {
       "name": "aho-corasick",

cargo-bazel-lock.json

@@ -11,8 +11,8 @@ stardoc_with_diff_test(
 )
 
 stardoc_with_diff_test(
-    name = "repositories",
-    bzl_library_target = "//oci:repositories",
+    name = "pull",
+    bzl_library_target = "//oci/repositories:pull",
 )
 
 update_docs()

docs/BUILD.bazel

@@ -106,35 +106,3 @@ oci_image_layer
 | <a id="oci_image_layer-kwargs"></a>kwargs |  Additional arguments to pass to the rule, e.g. `tags` or `visibility`   |  none |
 
 
-<a id="oci_pull"></a>
-
-## oci_pull
-
-<pre>
-oci_pull(<a href="#oci_pull-name">name</a>, <a href="#oci_pull-debug">debug</a>, <a href="#oci_pull-digest">digest</a>, <a href="#oci_pull-registry">registry</a>, <a href="#oci_pull-repo_mapping">repo_mapping</a>, <a href="#oci_pull-repository">repository</a>, <a href="#oci_pull-scheme">scheme</a>, <a href="#oci_pull-shallow">shallow</a>)
-</pre>
-
-**ATTRIBUTES**
-
-
-| Name  | Description | Type | Mandatory | Default |
-| :------------- | :------------- | :------------- | :------------- | :------------- |
-| <a id="oci_pull-name"></a>name |  A unique name for this repository.   | <a href="https://bazel.build/concepts/labels#target-names">Name</a> | required |  |
-| <a id="oci_pull-debug"></a>debug |  Deprecated. Does nothing   | Boolean | optional |  `False`  |
-| <a id="oci_pull-digest"></a>digest |  The digest or tag of the manifest file   | String | required |  |
-| <a id="oci_pull-registry"></a>registry |  Remote registry host to pull from, e.g. `gcr.io` or `index.docker.io`   | String | required |  |
-| <a id="oci_pull-repo_mapping"></a>repo_mapping |  In `WORKSPACE` context only: a dictionary from local repository name to global repository name. This allows controls over workspace dependency resolution for dependencies of this repository.<br><br>For example, an entry `"@foo": "@bar"` declares that, for any time this repository depends on `@foo` (such as a dependency on `@foo//some:target`, it should actually resolve that dependency within globally-declared `@bar` (`@bar//some:target`).<br><br>This attribute is _not_ supported in `MODULE.bazel` context (when invoking a repository rule inside a module extension's implementation function).   | <a href="https://bazel.build/rules/lib/dict">Dictionary: String -> String</a> | optional |  |
-| <a id="oci_pull-repository"></a>repository |  Image path beneath the registry, e.g. `distroless/static`   | String | required |  |
-| <a id="oci_pull-scheme"></a>scheme |  scheme portion of the URL for fetching from the registry   | String | optional |  `"https"`  |
-| <a id="oci_pull-shallow"></a>shallow |  Deprecated. Does nothing   | Boolean | optional |  `False`  |
-
-**ENVIRONMENT VARIABLES**
-
-This repository rule depends on the following environment variables:
-* `DOCKER_CONFIG`
-* `REGISTRY_AUTH_FILE`
-* `XDG_RUNTIME_DIR`
-* `HOME`
-* `OCI_ENABLE_OAUTH2_SUPPORT`
-
-