RustyV8: Example of creating a TypedArray?
Hey there, I'm currently trying to Zero-copy my
ort::Tensor to a v8::TypedArray, but I can't figure out how to work with the byte_offset and length parameters of v8::Float32Array or other TypedArray variant.
Here is an example of my code:
// ... model init
// ... v8 scope
let mut outputs = session.run(inputs![/* etc... */]);
// I do remove to get ownership of the value, discard the key since I don't need it
let (_key, last_hidden) = outputs.remove('last_hidden_state');
/** Then I get an `ArrayBuffer` from the inner `ptr` of my tensor **/
// I could try `Tensor::try_extract_raw_tensor_mut<T>` with `v8::ArrayBuffer::new_backing_store_from_bytes`
// but it only allows [u8] instead of [T], so we need to get into `unsafe` path.
let (_, raw_tensor) = last_hidden
.try_extract_raw_tensor_mut::<f32>()
.map_err(AnyError::from)?;
println!("tensor: len: {} -> {raw_tensor:?}", raw_tensor.len());
// $ "tensor: len: 3456 -> [0.22348395, 0.13147816, 0.32118404, 0.13782105, 0.10047962, -0.2339811, ...]"
// We zero-copy an ORT Tensor to a JS ArrayBuffer like:
// https://github.com/denoland/deno_core/blob/7258aa325368a8e2c1271a25c1b4d537ed41e9c5/core/runtime/ops_rust_to_v8.rs#L370
let tensor_ptr = raw_tensor.as_ptr();
let tensor_len = raw_tensor.len();
let tensor_rc = Rc::into_raw(Rc::new(raw_tensor)) as *const c_void;
extern "C" fn drop_tensor(_ptr: *mut c_void, _len: usize, data: *mut c_void) {
// SAFETY: We know that data is a raw Rc from above
unsafe { drop(Rc::from_raw(data.cast::<f32>())) }
}
let buf_store = unsafe {
v8::ArrayBuffer::new_backing_store_from_ptr(
tensor_ptr as _,
tensor_len,
drop_tensor,
tensor_rc as _,
)
}
.make_shared();
let array_buffer = v8::ArrayBuffer::with_backing_store(scope, &buf_store)
// ... model init
// ... v8 scope
let mut outputs = session.run(inputs![/* etc... */]);
// I do remove to get ownership of the value, discard the key since I don't need it
let (_key, last_hidden) = outputs.remove('last_hidden_state');
/** Then I get an `ArrayBuffer` from the inner `ptr` of my tensor **/
// I could try `Tensor::try_extract_raw_tensor_mut<T>` with `v8::ArrayBuffer::new_backing_store_from_bytes`
// but it only allows [u8] instead of [T], so we need to get into `unsafe` path.
let (_, raw_tensor) = last_hidden
.try_extract_raw_tensor_mut::<f32>()
.map_err(AnyError::from)?;
println!("tensor: len: {} -> {raw_tensor:?}", raw_tensor.len());
// $ "tensor: len: 3456 -> [0.22348395, 0.13147816, 0.32118404, 0.13782105, 0.10047962, -0.2339811, ...]"
// We zero-copy an ORT Tensor to a JS ArrayBuffer like:
// https://github.com/denoland/deno_core/blob/7258aa325368a8e2c1271a25c1b4d537ed41e9c5/core/runtime/ops_rust_to_v8.rs#L370
let tensor_ptr = raw_tensor.as_ptr();
let tensor_len = raw_tensor.len();
let tensor_rc = Rc::into_raw(Rc::new(raw_tensor)) as *const c_void;
extern "C" fn drop_tensor(_ptr: *mut c_void, _len: usize, data: *mut c_void) {
// SAFETY: We know that data is a raw Rc from above
unsafe { drop(Rc::from_raw(data.cast::<f32>())) }
}
let buf_store = unsafe {
v8::ArrayBuffer::new_backing_store_from_ptr(
tensor_ptr as _,
tensor_len,
drop_tensor,
tensor_rc as _,
)
}
.make_shared();
let array_buffer = v8::ArrayBuffer::with_backing_store(scope, &buf_store)
1 Reply
/** HERE is my problem: **/
// I don't now how to pass: `byte_offset` and `length`
/** TEST 1: Trying a length of 1: **/
let tensor_data = v8::Float32Array::new(scope, array_buffer , 0, 1).unwrap().into()
// JS output: { Float32Array(1) [ 0.22348394989967346 ] } -> But I was expecting an array of 3456 like the raw tensor above.
/** TEST 2: Trying the same length of original buffer [3456]: **/
let length = array_buffer.byte_length(); // $ 3456
let tensor_data = v8::Float32Array::new(scope, array_buffer , 0, length).unwrap().into()
// $ Runtime Error: "Check failed: byte_length <= buffer->GetByteLength()"
/** TEST 3: Trying buffer length -1: **/
let length = array_buffer.byte_length() -1; // $ 3455
let tensor_data = v8::Float32Array::new(scope, array_buffer , 0, length).unwrap().into()
// $ Runtime Error: "Check failed: byte_length <= buffer->GetByteLength()"
/** TEST 4: Trying to divide based on size: **/
let length = array_buffer.byte_length() / size_of::<f32>(); // $ 864
let tensor_data = v8::Float32Array::new(scope, array_buffer , 0, length).unwrap().into()
// JS output: { Float32Array(864) [ 0.22348395, 0.13147816, ... ] } -> But I was expecting an array of 3456 like the raw tensor above.
/** TEST 5: Trying to divide and offset based on size: **/
let byte_offset = size_of::<f32>(); // 4
let length = array_buffer.byte_length() / size_of::<f32>(); // $ 864
let tensor_data = v8::Float32Array::new(scope, array_buffer , byte_offset, length).unwrap().into()
// $ Runtime Error: Check failed: byte_offset + byte_length <= buffer->GetByteLength()
/** HERE is my problem: **/
// I don't now how to pass: `byte_offset` and `length`
/** TEST 1: Trying a length of 1: **/
let tensor_data = v8::Float32Array::new(scope, array_buffer , 0, 1).unwrap().into()
// JS output: { Float32Array(1) [ 0.22348394989967346 ] } -> But I was expecting an array of 3456 like the raw tensor above.
/** TEST 2: Trying the same length of original buffer [3456]: **/
let length = array_buffer.byte_length(); // $ 3456
let tensor_data = v8::Float32Array::new(scope, array_buffer , 0, length).unwrap().into()
// $ Runtime Error: "Check failed: byte_length <= buffer->GetByteLength()"
/** TEST 3: Trying buffer length -1: **/
let length = array_buffer.byte_length() -1; // $ 3455
let tensor_data = v8::Float32Array::new(scope, array_buffer , 0, length).unwrap().into()
// $ Runtime Error: "Check failed: byte_length <= buffer->GetByteLength()"
/** TEST 4: Trying to divide based on size: **/
let length = array_buffer.byte_length() / size_of::<f32>(); // $ 864
let tensor_data = v8::Float32Array::new(scope, array_buffer , 0, length).unwrap().into()
// JS output: { Float32Array(864) [ 0.22348395, 0.13147816, ... ] } -> But I was expecting an array of 3456 like the raw tensor above.
/** TEST 5: Trying to divide and offset based on size: **/
let byte_offset = size_of::<f32>(); // 4
let length = array_buffer.byte_length() / size_of::<f32>(); // $ 864
let tensor_data = v8::Float32Array::new(scope, array_buffer , byte_offset, length).unwrap().into()
// $ Runtime Error: Check failed: byte_offset + byte_length <= buffer->GetByteLength()
ArrayBuffer length as multiple from the current type:
let buf_store = unsafe {
v8::ArrayBuffer::new_backing_store_from_ptr(
tensor_ptr as _,
tensor_len * size_of::<f32>(), // Here I need to multiply based on type size
drop_tensor,
tensor_rc as _,
)
}
.make_shared();
let buf_store = unsafe {
v8::ArrayBuffer::new_backing_store_from_ptr(
tensor_ptr as _,
tensor_len * size_of::<f32>(), // Here I need to multiply based on type size
drop_tensor,
tensor_rc as _,
)
}
.make_shared();