Kalleby Santos Posts

Kalleby Santos

•Created by Kalleby Santos on 9/24/2024 in #help

RustyV8: Example of creating a TypedArray?

Hey there, I'm currently trying to Zero-copy my ort::Tensor to a v8::TypedArray, but I can't figure out how to work with the byte_offset and length parameters of v8::Float32Array or other TypedArray variant. Here is an example of my code:

// ... model init
// ... v8 scope

let mut outputs = session.run(inputs![/* etc... */]);

// I do remove to get ownership of the value, discard the key since I don't need it
let (_key, last_hidden) = outputs.remove('last_hidden_state');

/** Then I get an `ArrayBuffer` from the inner `ptr` of my tensor **/
// I could try `Tensor::try_extract_raw_tensor_mut<T>` with `v8::ArrayBuffer::new_backing_store_from_bytes`
// but it only allows [u8] instead of [T], so we need to get into `unsafe` path.
let (_, raw_tensor) = last_hidden
    .try_extract_raw_tensor_mut::<f32>()
    .map_err(AnyError::from)?;

println!("tensor: len: {} -> {raw_tensor:?}", raw_tensor.len());
// $ "tensor: len: 3456 -> [0.22348395, 0.13147816, 0.32118404, 0.13782105, 0.10047962, -0.2339811, ...]"

// We zero-copy an ORT Tensor to a JS ArrayBuffer like:
// https://github.com/denoland/deno_core/blob/7258aa325368a8e2c1271a25c1b4d537ed41e9c5/core/runtime/ops_rust_to_v8.rs#L370
let tensor_ptr = raw_tensor.as_ptr();
let tensor_len = raw_tensor.len();
let tensor_rc = Rc::into_raw(Rc::new(raw_tensor)) as *const c_void;

extern "C" fn drop_tensor(_ptr: *mut c_void, _len: usize, data: *mut c_void) {
    // SAFETY: We know that data is a raw Rc from above
    unsafe { drop(Rc::from_raw(data.cast::<f32>())) }
}

let buf_store = unsafe {
    v8::ArrayBuffer::new_backing_store_from_ptr(
        tensor_ptr as _,
        tensor_len,
        drop_tensor,
        tensor_rc as _,
    )
}
.make_shared();

let array_buffer = v8::ArrayBuffer::with_backing_store(scope, &buf_store)

// ... model init
// ... v8 scope

let mut outputs = session.run(inputs![/* etc... */]);

// I do remove to get ownership of the value, discard the key since I don't need it
let (_key, last_hidden) = outputs.remove('last_hidden_state');

/** Then I get an `ArrayBuffer` from the inner `ptr` of my tensor **/
// I could try `Tensor::try_extract_raw_tensor_mut<T>` with `v8::ArrayBuffer::new_backing_store_from_bytes`
// but it only allows [u8] instead of [T], so we need to get into `unsafe` path.
let (_, raw_tensor) = last_hidden
    .try_extract_raw_tensor_mut::<f32>()
    .map_err(AnyError::from)?;

println!("tensor: len: {} -> {raw_tensor:?}", raw_tensor.len());
// $ "tensor: len: 3456 -> [0.22348395, 0.13147816, 0.32118404, 0.13782105, 0.10047962, -0.2339811, ...]"

// We zero-copy an ORT Tensor to a JS ArrayBuffer like:
// https://github.com/denoland/deno_core/blob/7258aa325368a8e2c1271a25c1b4d537ed41e9c5/core/runtime/ops_rust_to_v8.rs#L370
let tensor_ptr = raw_tensor.as_ptr();
let tensor_len = raw_tensor.len();
let tensor_rc = Rc::into_raw(Rc::new(raw_tensor)) as *const c_void;

extern "C" fn drop_tensor(_ptr: *mut c_void, _len: usize, data: *mut c_void) {
    // SAFETY: We know that data is a raw Rc from above
    unsafe { drop(Rc::from_raw(data.cast::<f32>())) }
}

let buf_store = unsafe {
    v8::ArrayBuffer::new_backing_store_from_ptr(
        tensor_ptr as _,
        tensor_len,
        drop_tensor,
        tensor_rc as _,
    )
}
.make_shared();

let array_buffer = v8::ArrayBuffer::with_backing_store(scope, &buf_store)

3 replies

DDeno

•Created by Kalleby Santos on 9/12/2024 in #help

Embedding Deno: How to add external dependencies to esm file?

1 replies