VM Fuzzer (#2401)

This Pull Request offers a basic VM fuzzer which relies on implied oracles (namely, "does it crash or timeout?").

It changes the following:

- Adds an insns_remaining field to Context, denoting the number of instructions remaining to execute (only available when fuzzing)
- Adds a JsNativeError variant, denoting when the number of instructions has been exceeded (only available when fuzzing)
- Adds a VM fuzzer which looks for cases where Boa may crash on an input

This offers no guarantees about correctness, only assertion violations. Depends on #2400.

Any issues I raise in association with this fuzzer will link back to this fuzzer.

You may run the fuzzer using the following commands:
```bash
$ cd boa_engine
$ cargo +nightly fuzz run -s none vm-implied
```

Co-authored-by: Addison Crump <addison.crump@cispa.de>

boa_engine/src/context/mod.rs

@@ -97,6 +97,10 @@ pub struct Context {
     #[cfg(feature = "intl")]
     icu: icu::Icu,
 
+    /// Number of instructions remaining before a forced exit
+    #[cfg(feature = "fuzz")]
+    pub(crate) instructions_remaining: usize,
+
     pub(crate) vm: Vm,
 
     pub(crate) promise_job_queue: VecDeque<JobCallback>,
@@ -593,6 +597,8 @@ pub struct ContextBuilder {
     interner: Option<Interner>,
     #[cfg(feature = "intl")]
     icu: Option<icu::Icu>,
+    #[cfg(feature = "fuzz")]
+    instructions_remaining: usize,
 }
 
 impl ContextBuilder {
@@ -615,6 +621,15 @@ impl ContextBuilder {
         Ok(self)
     }
 
+    /// Specifies the number of instructions remaining to the [`Context`].
+    ///
+    /// This function is only available if the `fuzz` feature is enabled.
+    #[cfg(feature = "fuzz")]
+    pub fn instructions_remaining(mut self, instructions_remaining: usize) -> Self {
+        self.instructions_remaining = instructions_remaining;
+        self
+    }
+
     /// Creates a new [`ContextBuilder`] with a default empty [`Interner`]
     /// and a default [`BoaProvider`] if the `intl` feature is enabled.
     pub fn new() -> Self {
@@ -643,6 +658,8 @@ impl ContextBuilder {
                 icu::Icu::new(Box::new(icu_testdata::get_provider()))
                     .expect("Failed to initialize default icu data.")
             }),
+            #[cfg(feature = "fuzz")]
+            instructions_remaining: self.instructions_remaining,
             promise_job_queue: VecDeque::new(),
         };
 

boa_engine/src/error.rs

@@ -503,6 +503,17 @@ impl JsNativeError {
         Self::new(JsNativeErrorKind::Uri, Box::default(), None)
     }
 
+    /// Creates a new `JsNativeError` that indicates that the context hit its execution limit. This
+    /// is only used in a fuzzing context.
+    #[cfg(feature = "fuzz")]
+    pub fn no_instructions_remain() -> Self {
+        Self::new(
+            JsNativeErrorKind::NoInstructionsRemain,
+            Box::default(),
+            None,
+        )
+    }
+
     /// Sets the message of this error.
     ///
     /// # Examples
@@ -619,6 +630,12 @@ impl JsNativeError {
             }
             JsNativeErrorKind::Type => (constructors.type_error().prototype(), ErrorKind::Type),
             JsNativeErrorKind::Uri => (constructors.uri_error().prototype(), ErrorKind::Uri),
+            #[cfg(feature = "fuzz")]
+            JsNativeErrorKind::NoInstructionsRemain => {
+                unreachable!(
+                    "The NoInstructionsRemain native error cannot be converted to an opaque type."
+                )
+            }
         };
 
         let o = JsObject::from_proto_and_data(prototype, ObjectData::error(tag));
@@ -747,6 +764,10 @@ pub enum JsNativeErrorKind {
     /// [e_uri]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/encodeURI
     /// [d_uri]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/decodeURI
     Uri,
+    /// Error thrown when no instructions remain. Only used in a fuzzing context; not a valid JS
+    /// error variant.
+    #[cfg(feature = "fuzz")]
+    NoInstructionsRemain,
 }
 
 impl std::fmt::Display for JsNativeErrorKind {
@@ -760,6 +781,8 @@ impl std::fmt::Display for JsNativeErrorKind {
             JsNativeErrorKind::Syntax => "SyntaxError",
             JsNativeErrorKind::Type => "TypeError",
             JsNativeErrorKind::Uri => "UriError",
+            #[cfg(feature = "fuzz")]
+            JsNativeErrorKind::NoInstructionsRemain => "NoInstructionsRemain",
         }
         .fmt(f)
     }

boa_engine/src/vm/mod.rs

@@ -7,6 +7,8 @@ use crate::{
     vm::{call_frame::CatchAddresses, code_block::Readable},
     Context, JsResult, JsValue,
 };
+#[cfg(feature = "fuzz")]
+use crate::{JsError, JsNativeError};
 use boa_interner::ToInternedString;
 use boa_profiler::Profiler;
 use std::{convert::TryInto, mem::size_of, time::Instant};
@@ -179,6 +181,13 @@ impl Context {
             });
 
         while self.vm.frame().pc < self.vm.frame().code.code.len() {
+            #[cfg(feature = "fuzz")]
+            if self.instructions_remaining == 0 {
+                return Err(JsError::from_native(JsNativeError::no_instructions_remain()));
+            } else {
+                self.instructions_remaining -= 1;
+            }
+
             let result = if self.vm.trace {
                 let mut pc = self.vm.frame().pc;
                 let opcode: Opcode = self

fuzz/Cargo.lock

@@ -59,7 +59,6 @@ dependencies = [
  "chrono",
  "dyn-clone",
  "fast-float",
- "gc",
  "indexmap",
  "num-bigint",
  "num-integer",
@@ -93,7 +92,8 @@ dependencies = [
 name = "boa_gc"
 version = "0.16.0"
 dependencies = [
- "gc",
+ "boa_macros",
+ "boa_profiler",
 ]
 
 [[package]]
@@ -113,8 +113,10 @@ dependencies = [
 name = "boa_macros"
 version = "0.16.0"
 dependencies = [
+ "proc-macro2",
  "quote",
  "syn",
+ "synstructure",
 ]
 
 [[package]]
@@ -167,9 +169,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "chrono"
-version = "0.4.22"
+version = "0.4.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bfd4d1b31faaa3a89d7934dbded3111da0d2ef28e3ebccdb4f0179f5929d1ef1"
+checksum = "16b0a3d9ed01224b22057780a37bb8c5dbfe1be8ba48678e7bf57ec4b385411f"
 dependencies = [
  "iana-time-zone",
  "js-sys",
@@ -263,27 +265,6 @@ version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "95765f67b4b18863968b4a1bd5bb576f732b29a4a28c7cd84c09fa3e2875f33c"
 
-[[package]]
-name = "gc"
-version = "0.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3edaac0f5832202ebc99520cb77c932248010c4645d20be1dc62d6579f5b3752"
-dependencies = [
- "gc_derive",
-]
-
-[[package]]
-name = "gc_derive"
-version = "0.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "60df8444f094ff7885631d80e78eb7d88c3c2361a98daaabb06256e4500db941"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
- "synstructure",
-]
-
 [[package]]
 name = "getrandom"
 version = "0.2.8"

fuzz/Cargo.toml

@@ -27,3 +27,15 @@ name = "parser-idempotency"
 path = "fuzz_targets/parser-idempotency.rs"
 test = false
 doc = false
+
+[[bin]]
+name = "vm-implied"
+path = "fuzz_targets/vm-implied.rs"
+test = false
+doc = false
+
+[[bin]]
+name = "bytecompiler-implied"
+path = "fuzz_targets/bytecompiler-implied.rs"
+test = false
+doc = false

fuzz/README.md

@@ -35,3 +35,22 @@ following:
      information, as the inputs parsed between the two should be the same.
 
 In this way, this fuzzer can identify correctness issues present in the parser.
+
+## Bytecompiler Fuzzer
+
+The bytecompiler fuzzer, located in [bytecompiler-implied.rs](fuzz_targets/bytecompiler-implied.rs), identifies cases
+which cause an assertion failure in the bytecompiler. These crashes can cause denial of service issues and may block the
+discovery of crash cases in the VM fuzzer.
+
+## VM Fuzzer
+
+The VM fuzzer, located in [vm-implied.rs](fuzz_targets/vm-implied.rs), identifies crash cases in the VM. It does so by
+generating an arbitrary AST, converting it to source code (to remove invalid inputs), then executing that source code.
+Because we are not comparing against any invariants other than "does it crash", this fuzzer will only discover faults
+which cause the VM to terminate unexpectedly, e.g. as a result of a panic. It will not discover logic errors present in
+the VM.
+
+To ensure that the VM does not attempt to execute an infinite loop, Boa is restricted to a finite number of instructions
+before the VM is terminated. If a program takes more than a second or so to execute, it likely indicates an issue in the
+VM (as we expect the fuzzer to execute only a certain amount of instructions, which should take significantly less
+time).

fuzz/fuzz_targets/bytecompiler-implied.rs

@@ -0,0 +1,25 @@
+#![no_main]
+
+mod common;
+
+use crate::common::FuzzSource;
+use boa_engine::Context;
+use boa_parser::Parser;
+use libfuzzer_sys::{fuzz_target, Corpus};
+use std::io::Cursor;
+
+fn do_fuzz(original: FuzzSource) -> Corpus {
+    let mut ctx = Context::builder()
+        .interner(original.interner)
+        .instructions_remaining(0)
+        .build();
+    let mut parser = Parser::new(Cursor::new(&original.source));
+    if let Ok(parsed) = parser.parse_all(ctx.interner_mut()) {
+        let _ = ctx.compile(&parsed);
+        Corpus::Keep
+    } else {
+        Corpus::Reject
+    }
+}
+
+fuzz_target!(|original: FuzzSource| -> Corpus { do_fuzz(original) });

fuzz/fuzz_targets/common.rs

@@ -2,7 +2,7 @@ use boa_ast::{
     visitor::{VisitWith, VisitorMut},
     Expression, StatementList,
 };
-use boa_interner::{Interner, Sym};
+use boa_interner::{Interner, Sym, ToInternedString};
 use libfuzzer_sys::arbitrary;
 use libfuzzer_sys::arbitrary::{Arbitrary, Unstructured};
 use std::fmt::{Debug, Formatter};
@@ -72,3 +72,25 @@ impl Debug for FuzzData {
             .finish_non_exhaustive()
     }
 }
+
+pub struct FuzzSource {
+    pub interner: Interner,
+    pub source: String,
+}
+
+impl<'a> Arbitrary<'a> for FuzzSource {
+    fn arbitrary(u: &mut Unstructured<'a>) -> arbitrary::Result<Self> {
+        let data = FuzzData::arbitrary(u)?;
+        let source = data.ast.to_interned_string(&data.interner);
+        Ok(Self {
+            interner: data.interner,
+            source,
+        })
+    }
+}
+
+impl Debug for FuzzSource {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        f.write_fmt(format_args!("Fuzzed source:\n{}", self.source))
+    }
+}

fuzz/fuzz_targets/vm-implied.rs

@@ -0,0 +1,19 @@
+#![no_main]
+
+mod common;
+
+use crate::common::FuzzSource;
+use boa_engine::{Context, JsResult, JsValue};
+use libfuzzer_sys::fuzz_target;
+
+fn do_fuzz(original: FuzzSource) -> JsResult<JsValue> {
+    let mut ctx = Context::builder()
+        .interner(original.interner)
+        .instructions_remaining(1 << 16)
+        .build();
+    ctx.eval(&original.source)
+}
+
+fuzz_target!(|original: FuzzSource| {
+    let _ = do_fuzz(original);
+});