From 3f36cb2cfbcde57e28a01ca93c757df0c6415cb6 Mon Sep 17 00:00:00 2001 From: Sergey Bronnikov Date: Mon, 1 May 2023 11:03:21 +0300 Subject: [PATCH] Support integration with luzer The patch enables using luzer for fuzzing Lua projects in OSS-Fuzz. Usage: sudo python infra/helper.py build_fuzzers lua-example sudo python infra/helper.py check_build lua-example fuzz_basic sudo python infra/helper.py run_fuzzer lua-example fuzz_basic Closes #13782 --- .../new-project-guide/lua_lang.md | 185 ++++++++++++++++++ docs/getting-started/new_project_guide.md | 1 + infra/base-images/base-runner/Dockerfile | 1 + infra/base-images/base-runner/bad_build_check | 8 + .../base-runner/ubuntu-20-04.Dockerfile | 2 + .../base-runner/ubuntu-24-04.Dockerfile | 2 + projects/lua-example/Dockerfile | 34 ++++ projects/lua-example/build.sh | 42 ++++ projects/lua-example/compile_lua_fuzzer | 37 ++++ projects/lua-example/example_basic.lua | 39 ++++ projects/lua-example/project.yaml | 8 + 11 files changed, 359 insertions(+) create mode 100644 docs/getting-started/new-project-guide/lua_lang.md create mode 100644 projects/lua-example/Dockerfile create mode 100755 projects/lua-example/build.sh create mode 100755 projects/lua-example/compile_lua_fuzzer create mode 100644 projects/lua-example/example_basic.lua create mode 100644 projects/lua-example/project.yaml diff --git a/docs/getting-started/new-project-guide/lua_lang.md b/docs/getting-started/new-project-guide/lua_lang.md new file mode 100644 index 000000000000..93c252779522 --- /dev/null +++ b/docs/getting-started/new-project-guide/lua_lang.md @@ -0,0 +1,185 @@ +--- +layout: default +title: Integrating a Lua project +parent: Setting up a new project +grand_parent: Getting started +nav_order: 4 +permalink: /getting-started/new-project-guide/lua-lang/ +--- + +# Integrating a Lua project +{: .no_toc} + +- TOC +{:toc} +--- + +The process of integrating a project written in Lua with OSS-Fuzz +is similar to the general [Setting up a new project]({{ site.baseurl +}}/getting-started/new-project-guide/) process. The key specifics of +integrating a Lua project are outlined below. + +## luzer + +Lua fuzzing in OSS-Fuzz is powered by +[luzer](https://github.com/ligurio/luzer). As luzer operates +directly on the Lua source code level, it can be applied to any +project written in a language that can be transpiled into Lua, +such as [MoonScript](https://moonscript.org/), +[TypeScriptToLua](https://typescripttolua.github.io/), +[Fennel](https://fennel-lang.org/), and [Urn](https://urn-lang.com/). +Also, it supports fuzzing C/C++ extensions written for Lua. When +fuzzing native code, luzer can be used in combination with +Address Sanitizer or Undefined Behavior Sanitizer to catch extra bugs. + +## Project files + +### Example project + +We recommend viewing +[lua-example](https://github.com/google/oss-fuzz/tree/master/projects/lua-example) +as an example of a simple Lua fuzzing project. This example also +demonstrates how to use luzer's Fuzzed Data Provider. + +### project.yaml + +The `language` attribute must be specified as follows: + +```yaml +language: c +``` + +The only supported fuzzing engine is libFuzzer (`libfuzzer`). + +```yaml +fuzzing_engines: + - libfuzzer +sanitizers: + - none +``` + +There is nothing special for sanitizer support in OSS-Fuzz +infrastructure. luzer builds its own DSO with libFuzzer and +sanitizer and `compile_lua_fuzzer` (also managed by project) sets +it to `LD_PRELOAD` if required. + +### Dockerfile + +The Dockerfile should start by `FROM gcr.io/oss-fuzz-base/base-builder`. + +The OSS-Fuzz base Docker images come without any pre-installed +components required for Lua fuzzing. Apart from that, you should +usually need to build or install a Lua runtime, luzer module, +clone the project, set a `WORKDIR`, and copy any necessary files, +or install any project-specific dependencies here as you normally would. + +### Fuzzers + +In the simplest case, every fuzzer consists of a single Lua file that defines +a function `TestOneInput` and executes a function named `luzer.Fuzz()`. +An example fuzz target could thus be a file `fuzz_basic.lua` with contents: + +```lua +local parser = require("src.luacheck.parser") +local decoder = require("luacheck.decoder") +local luzer = require("luzer") + +local function TestOneInput(buf) + parser.parse(decoder.decode(buf)) +end + +local args = { + print_final_stats = 1, +} +luzer.Fuzz(TestOneInput, nil, args) +``` + +### compile_lua_fuzzer + +Unlike projects for other languages, the base image does not +include a script that generates a wrapper script that can be used +as a drop-in replacement for libFuzzer. + +Therefore, you need to add such a script yourself. This script +sets a relative path to Lua runtime that will be used for running +tests and the necessary environment variables (for example, `LUA_PATH`, +`LUA_CPATH` and `LD_PRELOAD`) and specifies the path directly to +the `.lua` file containing the test implementation. The script +`compile_lua_fuzzer` must accept the same command line flags as +libFuzzer-based tests. + +Note, the resulting wrapper scripts must contain the word "luarocks" +to pass checks by `bad_build_check` in continuous integration. + +Then, you can use the script `compile_lua_fuzzer` to build the fuzzers. +A usage example from the `lua-example` project is + +```shell +compile_lua_fuzzer lua fuzz_basic.lua +``` + +Arguments are: + +* a relative path to a Lua runtime name +* a relative path to the fuzzing test inside the OSS Fuzz project directory + +The `lua-example` projects includes an +[example](https://github.com/google/oss-fuzz/blob/master/projects/lua-example/compile_lua_fuzzer) +of such script. + +### build.sh + +The script is executed within the image built from your [Dockerfile](#Dockerfile). + +In general, this script should do the following: + +- Set up or build a Lua runtime. +- Set up or build required dependencies for your tests. +- Generate wrapper scripts for your tests using [compile_lua_fuzzer](#compile_lua_fuzzer). + +Resulting binaries, tests and their wrapper scripts, and a +directory with Luarocks dependencies should be placed in `$OUT`. + +Beware, when installing the luzer module, you need to set the +environment variable `OSS_FUZZ` to non-empty value, otherwise the +build may fail. + +The [lua-example](https://github.com/google/oss-fuzz/blob/master/projects/lua-example/build.sh) +project contains an example of a `build.sh` for a Lua projects. + +## FuzzedDataProvider + +luzer provides a Fuzzed Data Provider that is helpful for splitting +a fuzz input into multiple parts of various Lua types. Its +functionality is similar to +[Fuzzed Data Provider](https://github.com/google/fuzzing/blob/master/docs/split-inputs.md#fuzzed-data-provider) +available in LLVM. Learn about methods, provided by FDP in luzer, +in [documentation](https://github.com/ligurio/luzer/blob/master/docs/api.md#structure-aware-fuzzing). + +A fuzz target using the `FuzzedDataProvider` would look as follows: + +```lua +local luzer = require("luzer") + +local function TestOneInput(buf) + local fdp = luzer.FuzzedDataProvider(buf) + local str = fdp:consume_string(4) + + local b = {} + str:gsub(".", function(c) table.insert(b, c) end) + local count = 0 + if b[1] == "o" then count = count + 1 end + if b[2] == "o" then count = count + 1 end + if b[3] == "p" then count = count + 1 end + if b[4] == "s" then count = count + 1 end + + if count == 4 then assert(nil) end +end + +local args = { + only_ascii = 1, + print_pcs = 1, +} + +luzer.Fuzz(TestOneInput, nil, args) +``` diff --git a/docs/getting-started/new_project_guide.md b/docs/getting-started/new_project_guide.md index 1cdd23fa92ad..344b73940cae 100644 --- a/docs/getting-started/new_project_guide.md +++ b/docs/getting-started/new_project_guide.md @@ -102,6 +102,7 @@ Programming language the project is written in. Values you can specify include: * [`jvm` (Java, Kotlin, Scala and other JVM-based languages)]({{ site.baseurl }}//getting-started/new-project-guide/jvm-lang/) * [`swift`]({{ site.baseurl }}//getting-started/new-project-guide/swift-lang/) * [`javascript`]({{ site.baseurl }}//getting-started/new-project-guide/javascript-lang/) +* [`lua`]({{ site.baseurl }}//getting-started/new-project-guide/lua-lang/) ### primary_contact, auto_ccs {#primary} The primary contact and list of other contacts to be CCed. Each person listed gets access to ClusterFuzz, including crash reports and fuzzer statistics, and are auto-cced on new bugs filed in the OSS-Fuzz diff --git a/infra/base-images/base-runner/Dockerfile b/infra/base-images/base-runner/Dockerfile index 5f5f455f11f1..9a4302ea7368 100644 --- a/infra/base-images/base-runner/Dockerfile +++ b/infra/base-images/base-runner/Dockerfile @@ -117,6 +117,7 @@ COPY --from=base-ruby /usr/local/bin/gem /usr/local/bin/gem COPY --from=base-ruby /usr/local/lib/ruby /usr/local/lib/ruby COPY --from=base-ruby /usr/local/include/ruby-3.3.0 /usr/local/include/ruby-3.3.0 +RUN apt-get update && apt-get install -y luarocks # Do this last to make developing these files easier/faster due to caching. COPY bad_build_check \ diff --git a/infra/base-images/base-runner/bad_build_check b/infra/base-images/base-runner/bad_build_check index e3f4787e865e..87f0f58f2390 100755 --- a/infra/base-images/base-runner/bad_build_check +++ b/infra/base-images/base-runner/bad_build_check @@ -330,6 +330,11 @@ function check_mixed_sanitizers { return 0 fi + # luzer-based tests. + # Sanitizer runtime is loaded via LD_PRELOAD, so this check does + # not apply. + egrep luarocks $FUZZER && return 0; + # For fuzztest fuzzers point to the binary instead of launcher script. if [[ $FUZZER == *"@"* ]]; then FUZZER=(${FUZZER//@/ }[0]) @@ -426,6 +431,9 @@ function check_architecture { FUZZER=${FUZZER}.pkg fi + # luzer-based tests. + egrep luarocks $FUZZER && return 0; + # For fuzztest fuzzers point to the binary instead of launcher script. if [[ $FUZZER == *"@"* ]]; then FUZZER=(${FUZZER//@/ }[0]) diff --git a/infra/base-images/base-runner/ubuntu-20-04.Dockerfile b/infra/base-images/base-runner/ubuntu-20-04.Dockerfile index c421cb21a003..7207d8cb2c13 100644 --- a/infra/base-images/base-runner/ubuntu-20-04.Dockerfile +++ b/infra/base-images/base-runner/ubuntu-20-04.Dockerfile @@ -117,6 +117,8 @@ COPY --from=base-ruby /usr/local/bin/gem /usr/local/bin/gem COPY --from=base-ruby /usr/local/lib/ruby /usr/local/lib/ruby COPY --from=base-ruby /usr/local/include/ruby-3.3.0 /usr/local/include/ruby-3.3.0 +RUN apt-get update && apt-get install -y luarocks + # Do this last to make developing these files easier/faster due to caching. COPY bad_build_check \ coverage \ diff --git a/infra/base-images/base-runner/ubuntu-24-04.Dockerfile b/infra/base-images/base-runner/ubuntu-24-04.Dockerfile index 2e252a25ffd8..5addb421416b 100644 --- a/infra/base-images/base-runner/ubuntu-24-04.Dockerfile +++ b/infra/base-images/base-runner/ubuntu-24-04.Dockerfile @@ -117,6 +117,8 @@ COPY --from=base-ruby /usr/local/bin/gem /usr/local/bin/gem COPY --from=base-ruby /usr/local/lib/ruby /usr/local/lib/ruby COPY --from=base-ruby /usr/local/include/ruby-3.3.0 /usr/local/include/ruby-3.3.0 +RUN apt-get update && apt-get install -y luarocks + # Do this last to make developing these files easier/faster due to caching. COPY bad_build_check \ coverage \ diff --git a/projects/lua-example/Dockerfile b/projects/lua-example/Dockerfile new file mode 100644 index 000000000000..6696c211de6b --- /dev/null +++ b/projects/lua-example/Dockerfile @@ -0,0 +1,34 @@ +# Copyright 2023-2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ + +FROM gcr.io/oss-fuzz-base/base-builder + +RUN apt-get update && apt-get install -y build-essential make + +COPY build.sh $SRC/ +COPY compile_lua_fuzzer $SRC/ + +# For real projects, you would clone your repo in the next step. +RUN mkdir -p $SRC/example + +# Ideally, you have already configured fuzz tests in your repo so +# that they run (in luzer regression mode) as part of unit testing. +# Keeping the fuzz tests in sync with the source code ensures that +# they are adjusted continue to work after code changes. Here, +# we copy them into the example project directory. +COPY example_basic.lua $SRC/example/fuzz_basic.lua + +WORKDIR $SRC/example diff --git a/projects/lua-example/build.sh b/projects/lua-example/build.sh new file mode 100755 index 000000000000..fe6968175696 --- /dev/null +++ b/projects/lua-example/build.sh @@ -0,0 +1,42 @@ +#!/bin/bash -eu + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ + +# Finish execution if libFuzzer is not used, because luzer +# is libFuzzer-based. +if [[ "$FUZZING_ENGINE" != libfuzzer ]]; then + return +fi + +# Install dependencies: +# 1. The libraries required for building luzer module. +# 2. PUC Rio Lua executable, required for running tests itself. +# 3. The Lua package manager. +apt install -y cmake liblua5.1-0 liblua5.1-0-dev lua5.1 luarocks + +export OSS_FUZZ=1 +# Install Lua libraries. +luarocks install --lua-version 5.1 --server=https://luarocks.org/dev --tree=lua_modules luzer + +LUA_RUNTIME_NAME=lua + +# Build fuzzers. +$SRC/compile_lua_fuzzer $LUA_RUNTIME_NAME fuzz_basic.lua + +cp fuzz_basic.lua "$OUT/" +cp /usr/bin/lua5.1 "$OUT/$LUA_RUNTIME_NAME" +cp -R lua_modules "$OUT/" diff --git a/projects/lua-example/compile_lua_fuzzer b/projects/lua-example/compile_lua_fuzzer new file mode 100755 index 000000000000..43e55545e988 --- /dev/null +++ b/projects/lua-example/compile_lua_fuzzer @@ -0,0 +1,37 @@ +#!/bin/bash -eu +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ + +# The Lua runtime name. +lua_runtime=$1 +# Path to the fuzz target source file relative to the project's root. +fuzz_target=$2 + +fuzzer_basename=$(basename -s .lua "$fuzz_target") + +# Create an execution wrapper that executes luzer with the correct +# arguments. +echo "#!/bin/bash + +# LLVMFuzzerTestOneInput so that the wrapper script is recognized +# as a fuzz target for 'check_build'. +project_dir=\$(dirname \"\$0\") +eval \$(luarocks --tree lua_modules path) +LD_PRELOAD=\$project_dir/lua_modules/lib/lua/5.1/libfuzzer_with_asan.so \ +ASAN_OPTIONS=\$ASAN_OPTIONS:symbolize=1:external_symbolizer_path=\$project_dir/llvm-symbolizer:detect_leaks=0 \ +\$project_dir/$lua_runtime \$project_dir/$fuzz_target \$@" > "$OUT/$fuzzer_basename" + +chmod +x "$OUT/$fuzzer_basename" diff --git a/projects/lua-example/example_basic.lua b/projects/lua-example/example_basic.lua new file mode 100644 index 000000000000..7e6acdf29b2e --- /dev/null +++ b/projects/lua-example/example_basic.lua @@ -0,0 +1,39 @@ +-- Copyright 2026 Google LLC + +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the +-- License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +-- either express or implied. +-- See the License for the specific language governing permissions +-- and limitations under the License. + +local luzer = require("luzer") + +local function TestOneInput(buf) + local fdp = luzer.FuzzedDataProvider(buf) + local str = fdp:consume_string(4) + + local b = {} + str:gsub(".", function(c) table.insert(b, c) end) + local count = 0 + if b[1] == "o" then count = count + 1 end + if b[2] == "o" then count = count + 1 end + if b[3] == "p" then count = count + 1 end + if b[4] == "s" then count = count + 1 end + + if count == 4 then assert(nil) end +end + +local args = { + only_ascii = 1, + print_pcs = 1, +} + +luzer.Fuzz(TestOneInput, nil, args) diff --git a/projects/lua-example/project.yaml b/projects/lua-example/project.yaml new file mode 100644 index 000000000000..775d7162c3a8 --- /dev/null +++ b/projects/lua-example/project.yaml @@ -0,0 +1,8 @@ +homepage: https://github.com/ligurio/luzer +language: c +main_repo: https://github.com/ligurio/luzer +fuzzing_engines: + # luzer is libFuzzer-based, other engines are not supported. + - libfuzzer +vendor_ccs: + - estetus@gmail.com