diff --git a/src/literal.h b/src/literal.h index adb8e2f171e..335e117c147 100644 --- a/src/literal.h +++ b/src/literal.h @@ -19,6 +19,7 @@ #include #include +#include #include "support/bits.h" #include "support/hash.h" @@ -212,7 +213,7 @@ class Literal { } } - static Literal makeFromMemory(void* p, Type type); + static Literal makeFromMemory(const void* p, Type type); static Literal makeSignedMin(Type type) { switch (type.getBasic()) { @@ -312,6 +313,12 @@ class Literal { Name getFunc() const; std::shared_ptr getFuncData() const; std::shared_ptr getGCData() const; + size_t getNumElements() const; + Literal getElement(size_t index, bool signed_ = false) const; + void setElement(size_t index, Literal value); + bool isRawBytes() const; + const std::vector& getRawBytes() const; + std::vector& getRawBytes(); std::shared_ptr getExnData() const; std::shared_ptr getContData() const; @@ -784,28 +791,65 @@ std::ostream& operator<<(std::ostream& o, wasm::Literals literals); // A GC Struct, Array, or String is a set of values with a type saying how it // should be interpreted. struct GCData { - // The element or field values. - Literals values; + // The element or field values. Primitive numeric arrays use raw byte buffers + // (std::vector), while reference arrays, structs, strings, and other + // reference allocations use Literals. + std::variant, Literals> storage; // The descriptor, if it exists, or null. Literal desc; GCData(Literals&& values, const Literal& desc = Literal::makeNull(HeapType::none)) - : values(std::move(values)), desc(desc) {} + : storage(std::move(values)), desc(desc) {} + + GCData(std::vector&& data, + const Literal& desc = Literal::makeNull(HeapType::none)) + : storage(std::move(data)), desc(desc) {} + + bool isRawBytes() const { + return std::holds_alternative>(storage); + } + + const std::vector& getRawBytes() const { + return std::get>(storage); + } + + std::vector& getRawBytes() { + return std::get>(storage); + } + + const Literals& getLiterals() const { return std::get(storage); } + + Literals& getLiterals() { return std::get(storage); } }; +inline bool Literal::isRawBytes() const { + assert(isData()); + return gcData->isRawBytes(); +} + +inline const std::vector& Literal::getRawBytes() const { + assert(isData()); + return gcData->getRawBytes(); +} + +inline std::vector& Literal::getRawBytes() { + assert(isData()); + return gcData->getRawBytes(); +} + inline bool Literal::hasExternPayload() const { if (isNull()) { return false; } assert(type.getHeapType().isMaybeShared(HeapType::ext)); - return gcData->values[0].type == Type::i32; + return gcData->getLiterals()[0].type == Type::i32; } inline int32_t Literal::getExternPayload() const { assert(hasExternPayload()); - return gcData->values[0].geti32(); + return gcData->getLiterals()[0].geti32(); } } // namespace wasm @@ -869,7 +913,7 @@ template<> struct hash { return digest; } if (a.type.isString()) { - auto& values = a.getGCData()->values; + auto& values = a.getGCData()->getLiterals(); wasm::rehash(digest, values.size()); for (auto c : values) { wasm::rehash(digest, c.getInteger()); diff --git a/src/passes/Precompute.cpp b/src/passes/Precompute.cpp index 8f550bd27eb..a1357e6475c 100644 --- a/src/passes/Precompute.cpp +++ b/src/passes/Precompute.cpp @@ -1103,7 +1103,7 @@ struct Precompute // string. bool isValidUTF16Literal(const Literal& value) { bool expectLowSurrogate = false; - for (auto& v : value.getGCData()->values) { + for (auto& v : value.getGCData()->getLiterals()) { auto c = v.getInteger(); if (c >= 0xDC00 && c <= 0xDFFF) { if (expectLowSurrogate) { diff --git a/src/tools/wasm-ctor-eval.cpp b/src/tools/wasm-ctor-eval.cpp index f5b05c98220..9025832bc00 100644 --- a/src/tools/wasm-ctor-eval.cpp +++ b/src/tools/wasm-ctor-eval.cpp @@ -912,7 +912,6 @@ struct CtorEvalExternalInterface : EvallingModuleRunner::ExternalInterface { } else { // This is the first usage of this data. Generate a struct.new / // array.new for it. - auto& values = data->values; std::vector args; // The initial values for this allocation may themselves be GC @@ -934,8 +933,8 @@ struct CtorEvalExternalInterface : EvallingModuleRunner::ExternalInterface { definingGlobals[data] = DefiningGlobalInfo{definingGlobalName, type}; } - for (auto& value : values) { - auto* serialized = getSerialization(value); + for (size_t i = 0; i < value.getNumElements(); i++) { + auto* serialized = getSerialization(value.getElement(i)); if (!serialized) { return nullptr; } diff --git a/src/wasm-builder.h b/src/wasm-builder.h index 3b4cbb2d7ef..a5644ebf763 100644 --- a/src/wasm-builder.h +++ b/src/wasm-builder.h @@ -1504,7 +1504,7 @@ class Builder { // The string is already WTF-16, but we need to convert from `Literals` to // actual string. std::stringstream wtf16; - for (auto c : value.getGCData()->values) { + for (auto c : value.getGCData()->getLiterals()) { auto u = c.getInteger(); assert(u < 0x10000); wtf16 << uint8_t(u & 0xFF); diff --git a/src/wasm-interpreter.h b/src/wasm-interpreter.h index 08a1405ba22..7b330c41844 100644 --- a/src/wasm-interpreter.h +++ b/src/wasm-interpreter.h @@ -370,10 +370,11 @@ class ExpressionRunner : public OverriddenVisitor { // this function in LSan. // // This consumes the input |data| entirely. - Literal makeGCData(Literals&& data, + template + Literal makeGCData(T&& data, Type type, Literal desc = Literal::makeNull(HeapType::none)) { - auto allocation = std::make_shared(std::move(data), desc); + auto allocation = std::make_shared(std::forward(data), desc); #if __has_feature(leak_sanitizer) || __has_feature(address_sanitizer) // GC data with cycles will leak, since shared_ptrs do not handle cycles. // Binaryen is generally not used in long-running programs so we just ignore @@ -384,6 +385,23 @@ class ExpressionRunner : public OverriddenVisitor { return Literal(allocation, type.getHeapType()); } + Literal makeGCArray(Type type, + size_t num, + Literal desc = Literal::makeNull(HeapType::none)) { + auto heapType = type.getHeapType(); + const auto& element = heapType.getArray().element; + if (element.type.isRef()) { + Literals data(num); + auto zero = Literal::makeZero(element.type); + for (size_t i = 0; i < num; ++i) { + data[i] = zero; + } + return makeGCData(std::move(data), type, desc); + } + size_t elemBytes = element.getByteSize(); + return makeGCData(std::vector(num * elemBytes, 0), type, desc); + } + // Same as makeGCData but for ExnData. Literal makeExnData(Tag* tag, const Literals& payload) { auto allocation = std::make_shared(tag, payload); @@ -393,20 +411,6 @@ class ExpressionRunner : public OverriddenVisitor { return Literal(allocation); } - template - void writeBytes(T value, int numBytes, size_t index, Literals& values) { - if constexpr (std::is_same_v>) { - for (int i = 0; i < numBytes; ++i) { - values[index + i] = Literal(static_cast(value[i])); - } - } else { - for (int i = 0; i < numBytes; ++i) { - values[index + i] = - Literal(static_cast((value >> (i * 8)) & 0xff)); - } - } - } - static Literal applyRMW(AtomicRMWOp op, Literal lhs, Literal rhs) { switch (op) { case RMWAdd: @@ -2276,7 +2280,8 @@ class ExpressionRunner : public OverriddenVisitor { trap("null ref"); } auto field = curr->ref->type.getHeapType().getStruct().fields[curr->index]; - return extendForPacking(data->values[curr->index], field, curr->signed_); + return extendForPacking( + data->getLiterals()[curr->index], field, curr->signed_); } Flow visitStructSet(StructSet* curr) { VISIT(ref, curr->ref) @@ -2286,7 +2291,7 @@ class ExpressionRunner : public OverriddenVisitor { trap("null ref"); } auto field = curr->ref->type.getHeapType().getStruct().fields[curr->index]; - data->values[curr->index] = + data->getLiterals()[curr->index] = truncateForPacking(value.getSingleValue(), field); return Flow(); } @@ -2298,7 +2303,7 @@ class ExpressionRunner : public OverriddenVisitor { if (!data) { trap("null ref"); } - auto& field = data->values[curr->index]; + auto& field = data->getLiterals()[curr->index]; auto oldVal = field; field = applyRMW(curr->op, oldVal, value.getSingleValue()); return oldVal; @@ -2312,7 +2317,7 @@ class ExpressionRunner : public OverriddenVisitor { if (!data) { trap("null ref"); } - auto& field = data->values[curr->index]; + auto& field = data->getLiterals()[curr->index]; auto oldVal = field; if (field == expected.getSingleValue()) { field = replacement.getSingleValue(); @@ -2337,7 +2342,7 @@ class ExpressionRunner : public OverriddenVisitor { if (!waitqueue.getSingleValue().getGCData()) { trap("null ref"); } - auto& field = data->values[curr->index]; + auto& field = data->getLiterals()[curr->index]; if (field != expected.getSingleValue()) { return Literal(int32_t{1}); // not equal } @@ -2354,7 +2359,7 @@ class ExpressionRunner : public OverriddenVisitor { } Flow visitWaitqueueNew(WaitqueueNew* curr) { - return self()->makeGCData({}, + return self()->makeGCData(Literals{}, Type(HeapTypes::sharedWaitqueue, NonNullable)); } @@ -2389,26 +2394,18 @@ class ExpressionRunner : public OverriddenVisitor { assert(init.breaking()); return init; } - auto heapType = curr->type.getHeapType(); - const auto& element = heapType.getArray().element; Index num = size.getSingleValue().geti32(); if (num >= DataLimit) { hostLimit("allocation failure"); } - Literals data(num); - if (curr->isWithDefault()) { - auto zero = Literal::makeZero(element.type); - for (Index i = 0; i < num; i++) { - data[i] = zero; - } - } else { - auto field = curr->type.getHeapType().getArray().element; - auto value = truncateForPacking(init.getSingleValue(), field); + auto ret = makeGCArray(curr->type, num); + if (!curr->isWithDefault()) { + Literal val = init.getSingleValue(); for (Index i = 0; i < num; i++) { - data[i] = value; + ret.setElement(i, val); } } - return makeGCData(std::move(data), curr->type); + return ret; } Flow visitArrayNewData(ArrayNewData* curr) { WASM_UNREACHABLE("unimp"); } Flow visitArrayNewElem(ArrayNewElem* curr) { WASM_UNREACHABLE("unimp"); } @@ -2425,91 +2422,105 @@ class ExpressionRunner : public OverriddenVisitor { } WASM_UNREACHABLE("unreachable but no unreachable child"); } - auto heapType = curr->type.getHeapType(); - auto field = heapType.getArray().element; - Literals data(num); + auto ret = makeGCArray(curr->type, num); for (Index i = 0; i < num; i++) { VISIT(value, curr->values[i]) - data[i] = truncateForPacking(value.getSingleValue(), field); + ret.setElement(i, value.getSingleValue()); } - return makeGCData(std::move(data), curr->type); + return ret; } Flow visitArrayGet(ArrayGet* curr) { VISIT(ref, curr->ref) VISIT(index, curr->index) - auto data = ref.getSingleValue().getGCData(); - if (!data) { + auto refVal = ref.getSingleValue(); + if (refVal.isNull()) { trap("null ref"); } Index i = index.getSingleValue().geti32(); - if (i >= data->values.size()) { + if (i >= refVal.getNumElements()) { trap("array oob"); } - auto field = curr->ref->type.getHeapType().getArray().element; - return extendForPacking(data->values[i], field, curr->signed_); + return refVal.getElement(i, curr->signed_); } Flow visitArraySet(ArraySet* curr) { VISIT(ref, curr->ref) VISIT(index, curr->index) VISIT(value, curr->value) - auto data = ref.getSingleValue().getGCData(); - if (!data) { + auto refVal = ref.getSingleValue(); + if (refVal.isNull()) { trap("null ref"); } Index i = index.getSingleValue().geti32(); - if (i >= data->values.size()) { + if (i >= refVal.getNumElements()) { trap("array oob"); } - auto field = curr->ref->type.getHeapType().getArray().element; - data->values[i] = truncateForPacking(value.getSingleValue(), field); + refVal.setElement(i, value.getSingleValue()); return Flow(); } Flow visitArrayLoad(ArrayLoad* curr) { VISIT(ref, curr->ref) VISIT(index, curr->index) - auto data = ref.getSingleValue().getGCData(); - if (!data) { + auto refVal = ref.getSingleValue(); + if (refVal.isNull()) { trap("null ref"); } Index i = index.getSingleValue().geti32(); - size_t size = data->values.size(); + size_t size = refVal.getRawBytes().size(); if (i >= size || curr->bytes > (size - i)) { trap("array oob"); } - uint64_t val = 0; - for (unsigned b = 0; b < curr->bytes; ++b) { - val |= static_cast(data->values[i + b].geti32()) << (b * 8); - } + const uint8_t* p = &refVal.getRawBytes()[i]; switch (curr->type.getBasic()) { case Type::i32: { - int32_t sval = static_cast(val); - if (curr->signed_) { - if (curr->bytes == 1) { - sval = static_cast(static_cast(sval)); - } else if (curr->bytes == 2) { - sval = static_cast(static_cast(sval)); - } + switch (curr->bytes) { + case 1: + return curr->signed_ ? Literal((int32_t)Bits::readLE(p)) + : Literal((int32_t)Bits::readLE(p)); + case 2: + return curr->signed_ ? Literal((int32_t)Bits::readLE(p)) + : Literal((int32_t)Bits::readLE(p)); + case 4: + return Literal(Bits::readLE(p)); + default: + WASM_UNREACHABLE("invalid size"); } - return Literal(sval); } case Type::i64: { - int64_t sval = static_cast(val); - if (curr->signed_) { - if (curr->bytes == 1) { - sval = static_cast(static_cast(sval)); - } else if (curr->bytes == 2) { - sval = static_cast(static_cast(sval)); - } else if (curr->bytes == 4) { - sval = static_cast(static_cast(sval)); - } + switch (curr->bytes) { + case 1: + return curr->signed_ ? Literal((int64_t)Bits::readLE(p)) + : Literal((int64_t)Bits::readLE(p)); + case 2: + return curr->signed_ ? Literal((int64_t)Bits::readLE(p)) + : Literal((int64_t)Bits::readLE(p)); + case 4: + return curr->signed_ ? Literal((int64_t)Bits::readLE(p)) + : Literal((int64_t)Bits::readLE(p)); + case 8: + return Literal(Bits::readLE(p)); + default: + WASM_UNREACHABLE("invalid size"); } - return Literal(sval); } case Type::f32: { - return Literal(bit_cast(static_cast(val))); + switch (curr->bytes) { + case 2: + return Literal(bit_cast(fp16_ieee_to_fp32_value( + Bits::readLE(p)))) + .castToF32(); + case 4: + return Literal(Bits::readLE(p)).castToF32(); + default: + WASM_UNREACHABLE("invalid size"); + } } case Type::f64: { - return Literal(bit_cast(static_cast(val))); + assert(curr->bytes == 8); + return Literal(Bits::readLE(p)).castToF64(); + } + case Type::v128: { + assert(curr->bytes == 16); + return Literal(p); } default: WASM_UNREACHABLE("invalid type"); @@ -2520,55 +2531,36 @@ class ExpressionRunner : public OverriddenVisitor { VISIT(ref, curr->ref) VISIT(index, curr->index) VISIT(value, curr->value) - auto data = ref.getSingleValue().getGCData(); - if (!data) { + auto refVal = ref.getSingleValue(); + if (refVal.isNull()) { trap("null ref"); } Index i = index.getSingleValue().geti32(); - size_t size = data->values.size(); + size_t size = refVal.getRawBytes().size(); // Use subtraction to avoid overflow. if (i >= size || curr->bytes > (size - i)) { trap("array oob"); } - switch (curr->value->type.getBasic()) { - case Type::i32: - writeBytes( - value.getSingleValue().geti32(), curr->bytes, i, data->values); - break; - case Type::i64: - writeBytes( - value.getSingleValue().geti64(), curr->bytes, i, data->values); - break; - case Type::f32: - writeBytes(value.getSingleValue().reinterpreti32(), - curr->bytes, - i, - data->values); - break; - case Type::f64: - writeBytes(value.getSingleValue().reinterpreti64(), - curr->bytes, - i, - data->values); - break; - case Type::v128: - writeBytes( - value.getSingleValue().getv128(), curr->bytes, i, data->values); - break; - case Type::none: - case Type::unreachable: - WASM_UNREACHABLE("unimp basic type"); + uint8_t* p = &refVal.getRawBytes()[i]; + auto val = value.getSingleValue(); + if (curr->value->type == Type::f32 && curr->bytes == 2) { + float f32 = bit_cast(val.reinterpreti32()); + Bits::writeLE(fp16_ieee_from_fp32_value(f32), p); + } else { + uint8_t buf[16]; + val.getBits(buf); + memcpy(p, buf, curr->bytes); } return Flow(); } Flow visitArrayLen(ArrayLen* curr) { VISIT(ref, curr->ref) - auto data = ref.getSingleValue().getGCData(); - if (!data) { + auto refVal = ref.getSingleValue(); + if (refVal.isNull()) { trap("null ref"); } - return Literal(int32_t(data->values.size())); + return Literal(int32_t(refVal.getNumElements())); } Flow visitArrayCopy(ArrayCopy* curr) { VISIT(destRef, curr->destRef) @@ -2576,30 +2568,42 @@ class ExpressionRunner : public OverriddenVisitor { VISIT(srcRef, curr->srcRef) VISIT(srcIndex, curr->srcIndex) VISIT(length, curr->length) - auto destData = destRef.getSingleValue().getGCData(); - if (!destData) { + auto destVal = destRef.getSingleValue(); + if (destVal.isNull()) { trap("null ref"); } - auto srcData = srcRef.getSingleValue().getGCData(); - if (!srcData) { + auto srcVal = srcRef.getSingleValue(); + if (srcVal.isNull()) { trap("null ref"); } - size_t destVal = destIndex.getSingleValue().getUnsigned(); - size_t srcVal = srcIndex.getSingleValue().getUnsigned(); + size_t destIdx = destIndex.getSingleValue().getUnsigned(); + size_t srcIdx = srcIndex.getSingleValue().getUnsigned(); size_t lengthVal = length.getSingleValue().getUnsigned(); - if (destVal + lengthVal > destData->values.size()) { + if (destIdx + lengthVal > destVal.getNumElements()) { trap("oob"); } - if (srcVal + lengthVal > srcData->values.size()) { + if (srcIdx + lengthVal > srcVal.getNumElements()) { trap("oob"); } + // Fast path: bulk copy raw byte buffers directly with memmove rather than + // extracting and setting elements one-by-one via Literals. + if (destVal.isRawBytes() && srcVal.isRawBytes()) { + if (lengthVal > 0) { + auto elemSize = + destVal.type.getHeapType().getArray().element.getByteSize(); + memmove(&destVal.getRawBytes()[destIdx * elemSize], + &srcVal.getRawBytes()[srcIdx * elemSize], + lengthVal * elemSize); + } + return Flow(); + } std::vector copied; copied.resize(lengthVal); for (size_t i = 0; i < lengthVal; i++) { - copied[i] = srcData->values[srcVal + i]; + copied[i] = srcVal.getElement(srcIdx + i); } for (size_t i = 0; i < lengthVal; i++) { - destData->values[destVal + i] = copied[i]; + destVal.setElement(destIdx + i, copied[i]); } return Flow(); } @@ -2608,24 +2612,21 @@ class ExpressionRunner : public OverriddenVisitor { VISIT(index, curr->index) VISIT(value, curr->value) VISIT(size, curr->size) - auto data = ref.getSingleValue().getGCData(); - if (!data) { + auto refVal = ref.getSingleValue(); + if (refVal.isNull()) { trap("null ref"); } size_t indexVal = index.getSingleValue().getUnsigned(); Literal fillVal = value.getSingleValue(); size_t sizeVal = size.getSingleValue().getUnsigned(); - auto field = curr->ref->type.getHeapType().getArray().element; - fillVal = truncateForPacking(fillVal, field); - - size_t arraySize = data->values.size(); + size_t arraySize = refVal.getNumElements(); if (indexVal > arraySize || sizeVal > arraySize || indexVal + sizeVal > arraySize || indexVal + sizeVal < indexVal) { trap("out of bounds array access in array.fill"); } for (size_t i = 0; i < sizeVal; ++i) { - data->values[indexVal + i] = fillVal; + refVal.setElement(indexVal + i, fillVal); } return {}; } @@ -2635,17 +2636,17 @@ class ExpressionRunner : public OverriddenVisitor { VISIT(ref, curr->ref) VISIT(index, curr->index) VISIT(value, curr->value) - auto data = ref.getSingleValue().getGCData(); - if (!data) { + auto refVal = ref.getSingleValue(); + if (refVal.isNull()) { trap("null ref"); } size_t indexVal = index.getSingleValue().getUnsigned(); - if (indexVal >= data->values.size()) { + if (indexVal >= refVal.getNumElements()) { trap("array oob"); } - auto& field = data->values[indexVal]; - auto oldVal = field; - field = applyRMW(curr->op, oldVal, value.getSingleValue()); + auto oldVal = refVal.getElement(indexVal); + refVal.setElement(indexVal, + applyRMW(curr->op, oldVal, value.getSingleValue())); return oldVal; } @@ -2654,18 +2655,17 @@ class ExpressionRunner : public OverriddenVisitor { VISIT(index, curr->index) VISIT(expected, curr->expected) VISIT(replacement, curr->replacement) - auto data = ref.getSingleValue().getGCData(); - if (!data) { + auto refVal = ref.getSingleValue(); + if (refVal.isNull()) { trap("null ref"); } size_t indexVal = index.getSingleValue().getUnsigned(); - if (indexVal >= data->values.size()) { + if (indexVal >= refVal.getNumElements()) { trap("array oob"); } - auto& field = data->values[indexVal]; - auto oldVal = field; - if (field == expected.getSingleValue()) { - field = replacement.getSingleValue(); + auto oldVal = refVal.getElement(indexVal); + if (oldVal == expected.getSingleValue()) { + refVal.setElement(indexVal, replacement.getSingleValue()); } return oldVal; } @@ -2691,22 +2691,21 @@ class ExpressionRunner : public OverriddenVisitor { case StringNewWTF16Array: { VISIT(start, curr->start) VISIT(end, curr->end) - auto ptrData = ptr.getSingleValue().getGCData(); - if (!ptrData) { + auto ptrVal = ptr.getSingleValue(); + if (ptrVal.isNull()) { trap("null ref"); } - const auto& ptrDataValues = ptrData->values; + size_t arrayLen = ptrVal.getNumElements(); size_t startVal = start.getSingleValue().getUnsigned(); size_t endVal = end.getSingleValue().getUnsigned(); - if (startVal > ptrDataValues.size() || endVal > ptrDataValues.size() || - endVal < startVal) { + if (startVal > arrayLen || endVal > arrayLen || endVal < startVal) { trap("array oob"); } Literals contents; if (endVal > startVal) { contents.reserve(endVal - startVal); for (size_t i = startVal; i < endVal; i++) { - contents.push_back(ptrDataValues[i]); + contents.push_back(ptrVal.getElement(i)); } } return makeGCData(std::move(contents), curr->type); @@ -2743,7 +2742,7 @@ class ExpressionRunner : public OverriddenVisitor { trap("null ref"); } - return Literal(int32_t(data->values.size())); + return Literal(int32_t(data->getLiterals().size())); } Flow visitStringConcat(StringConcat* curr) { VISIT(flow, curr->left) @@ -2756,17 +2755,19 @@ class ExpressionRunner : public OverriddenVisitor { trap("null ref"); } - auto totalSize = leftData->values.size() + rightData->values.size(); + auto totalSize = + leftData->getLiterals().size() + rightData->getLiterals().size(); if (totalSize >= DataLimit) { hostLimit("allocation failure"); } Literals contents; - contents.reserve(leftData->values.size() + rightData->values.size()); - for (Literal& l : leftData->values) { + contents.reserve(leftData->getLiterals().size() + + rightData->getLiterals().size()); + for (Literal& l : leftData->getLiterals()) { contents.push_back(l); } - for (Literal& l : rightData->values) { + for (Literal& l : rightData->getLiterals()) { contents.push_back(l); } @@ -2783,24 +2784,24 @@ class ExpressionRunner : public OverriddenVisitor { VISIT(start, curr->start) auto strData = str.getSingleValue().getGCData(); - auto arrayData = array.getSingleValue().getGCData(); - if (!strData || !arrayData) { + auto arrayVal = array.getSingleValue(); + if (!strData || arrayVal.isNull()) { trap("null ref"); } auto startVal = start.getSingleValue().getUnsigned(); - auto& strValues = strData->values; - auto& arrayValues = arrayData->values; + auto& strValues = strData->getLiterals(); + size_t arrayLen = arrayVal.getNumElements(); size_t end; if (std::ckd_add(&end, startVal, strValues.size()) || - end > arrayValues.size()) { + end > arrayLen) { trap("oob"); } for (Index i = 0; i < strValues.size(); i++) { - arrayValues[startVal + i] = strValues[i]; + arrayVal.setElement(startVal + i, strValues[i]); } - return Literal(int32_t(strData->values.size())); + return Literal(int32_t(strValues.size())); } Flow visitStringEq(StringEq* curr) { VISIT(flow, curr->left) @@ -2813,17 +2814,17 @@ class ExpressionRunner : public OverriddenVisitor { switch (curr->op) { case StringEqEqual: { // They are equal if both are null, or both are non-null and equal. - result = - (!leftData && !rightData) || - (leftData && rightData && leftData->values == rightData->values); + result = (!leftData && !rightData) || + (leftData && rightData && + leftData->getLiterals() == rightData->getLiterals()); break; } case StringEqCompare: { if (!leftData || !rightData) { trap("null ref"); } - auto& leftValues = leftData->values; - auto& rightValues = rightData->values; + auto& leftValues = leftData->getLiterals(); + auto& rightValues = rightData->getLiterals(); Index i = 0; while (1) { if (i == leftValues.size() && i == rightValues.size()) { @@ -2873,7 +2874,7 @@ class ExpressionRunner : public OverriddenVisitor { if (!data) { trap("null ref"); } - auto& values = data->values; + auto& values = data->getLiterals(); Index i = pos.getSingleValue().geti32(); if (i >= values.size()) { trap("string oob"); @@ -2890,7 +2891,7 @@ class ExpressionRunner : public OverriddenVisitor { if (!refData) { trap("null ref"); } - auto& refValues = refData->values; + auto& refValues = refData->getLiterals(); auto startVal = start.getSingleValue().getUnsigned(); auto endVal = end.getSingleValue().getUnsigned(); endVal = std::min(endVal, refValues.size()); @@ -2950,22 +2951,6 @@ class ExpressionRunner : public OverriddenVisitor { } return value; } - - Literal makeFromMemory(void* p, Field field) { - switch (field.packedType) { - case Field::NotPacked: - return Literal::makeFromMemory(p, field.type); - case Field::i8: { - return truncateForPacking(Literal(int32_t(Bits::readLE(p))), - field); - } - case Field::i16: { - return truncateForPacking(Literal(int32_t(Bits::readLE(p))), - field); - } - } - WASM_UNREACHABLE("unexpected type"); - } }; // Execute a suspected constant expression (precompute and C-API). @@ -4736,7 +4721,6 @@ class ModuleRunnerBase : public ExpressionRunner { auto heapType = curr->type.getHeapType(); const auto& element = heapType.getArray().element; - Literals contents; const auto& seg = *wasm.getDataSegment(curr->segment); auto elemBytes = element.getByteSize(); @@ -4751,10 +4735,9 @@ class ModuleRunnerBase : public ExpressionRunner { if (droppedDataSegments.contains(curr->segment) && end > 0) { trap("dropped segment access in array.new_data"); } - contents.reserve(size); - for (Index i = offset; i < end; i += elemBytes) { - auto addr = (void*)&seg.data[i]; - contents.push_back(this->makeFromMemory(addr, element)); + std::vector contents(size * elemBytes); + if (size > 0) { + memcpy(contents.data(), &seg.data[offset], size * elemBytes); } #pragma GCC diagnostic pop @@ -4790,15 +4773,15 @@ class ModuleRunnerBase : public ExpressionRunner { VISIT(index, curr->index) VISIT(offset, curr->offset) VISIT(size, curr->size) - auto data = ref.getSingleValue().getGCData(); - if (!data) { + auto refVal = ref.getSingleValue(); + if (refVal.isNull()) { trap("null ref"); } size_t indexVal = index.getSingleValue().getUnsigned(); size_t offsetVal = offset.getSingleValue().getUnsigned(); size_t sizeVal = size.getSingleValue().getUnsigned(); - size_t arraySize = data->values.size(); + size_t arraySize = refVal.getNumElements(); if ((uint64_t)indexVal + sizeVal > arraySize) { trap("out of bounds array access in array.init"); } @@ -4816,9 +4799,11 @@ class ModuleRunnerBase : public ExpressionRunner { droppedDataSegments.contains(curr->segment)) { trap("out of bounds segment access in array.init_data"); } - for (size_t i = 0; i < sizeVal; i++) { - void* addr = (void*)&seg->data[offsetVal + i * elemSize]; - data->values[indexVal + i] = this->makeFromMemory(addr, elem); + assert(refVal.isRawBytes()); + if (sizeVal > 0) { + memcpy(&refVal.getRawBytes()[indexVal * elemSize], + &seg->data[offsetVal], + sizeVal * elemSize); } return {}; } @@ -4827,15 +4812,15 @@ class ModuleRunnerBase : public ExpressionRunner { VISIT(index, curr->index) VISIT(offset, curr->offset) VISIT(size, curr->size) - auto data = ref.getSingleValue().getGCData(); - if (!data) { + auto refVal = ref.getSingleValue(); + if (refVal.isNull()) { trap("null ref"); } size_t indexVal = index.getSingleValue().getUnsigned(); size_t offsetVal = offset.getSingleValue().getUnsigned(); size_t sizeVal = size.getSingleValue().getUnsigned(); - size_t arraySize = data->values.size(); + size_t arraySize = refVal.getNumElements(); if ((uint64_t)indexVal + sizeVal > arraySize) { trap("out of bounds array access in array.init"); } @@ -4855,7 +4840,8 @@ class ModuleRunnerBase : public ExpressionRunner { // of references in the table! ArrayNew suffers the same problem. // Fixing it will require changing how we represent segments, at least // in the interpreter. - data->values[indexVal + i] = self()->visit(seg->data[i]).getSingleValue(); + refVal.setElement(indexVal + i, + self()->visit(seg->data[i]).getSingleValue()); } return {}; } diff --git a/src/wasm/literal.cpp b/src/wasm/literal.cpp index a037fef99e6..272cc5a6404 100644 --- a/src/wasm/literal.cpp +++ b/src/wasm/literal.cpp @@ -314,7 +314,7 @@ Literal Literal::makeNegOne(Type type) { return makeFromInt32(-1, type); } -Literal Literal::makeFromMemory(void* p, Type type) { +Literal Literal::makeFromMemory(const void* p, Type type) { assert(type.isNumber()); switch (type.getBasic()) { case Type::i32: { @@ -499,7 +499,7 @@ bool Literal::operator==(const Literal& other) const { return *funcData == *other.funcData; } if (type.isString()) { - return gcData->values == other.gcData->values; + return gcData->getLiterals() == other.gcData->getLiterals(); } if (type.isData()) { return gcData == other.gcData; @@ -775,8 +775,8 @@ std::ostream& operator<<(std::ostream& o, Literal literal) { break; case HeapType::any: { auto data = literal.getGCData(); - assert(data->values.size() == 1); - o << "internalized " << literal.getGCData()->values[0]; + assert(data->getLiterals().size() == 1); + o << "internalized " << data->getLiterals()[0]; break; } case HeapType::ext: { @@ -806,7 +806,7 @@ std::ostream& operator<<(std::ostream& o, Literal literal) { o << "string("; // Convert WTF-16 literals to WTF-16 string. std::stringstream wtf16; - for (auto c : data->values) { + for (auto c : data->getLiterals()) { auto u = c.getInteger(); assert(u < 0x10000); wtf16 << uint8_t(u & 0xFF); @@ -844,9 +844,15 @@ std::ostream& operator<<(std::ostream& o, Literal literal) { assert(literal.isData()); auto data = literal.getGCData(); assert(data); - o << "[ref " << literal.type.getHeapType() << ' ' << data->values; + o << "[ref " << literal.type.getHeapType() << ' '; + for (size_t i = 0; i < literal.getNumElements(); i++) { + if (i > 0) { + o << ' '; + } + o << literal.getElement(i); + } if (!data->desc.isNull()) { - if (!data->values.empty()) { + if (literal.getNumElements() > 0) { o << ", "; } o << "desc=" << data->desc; @@ -3102,8 +3108,8 @@ Literal Literal::externalize() const { } if (heapType.isMaybeShared(HeapType::any)) { // This is an internalized externref or string; just unwrap it. - assert(gcData->values.size() == 1); - return gcData->values[0]; + assert(gcData->getLiterals().size() == 1); + return gcData->getLiterals()[0]; } // This is an internal reference. Wrap it. auto ext = HeapTypes::ext.getBasic(heapType.getShared()); @@ -3124,8 +3130,8 @@ Literal Literal::internalize() const { return Literal(std::make_shared(Literals{*this}), any); } // This is an externalized internal reference; just unwrap it. - assert(gcData->values.size() == 1); - return gcData->values[0]; + assert(gcData->getLiterals().size() == 1); + return gcData->getLiterals()[0]; } Literal Literal::unwrap() const { @@ -3148,7 +3154,7 @@ Literal Literal::getJSPrototype() const { assert(type.isRef()); if (auto desc = type.getHeapType().getDescriptorType(); desc && JSUtils::hasPossibleJSPrototypeField(*desc)) { - auto proto = gcData->desc.getGCData()->values[0].unwrap(); + auto proto = gcData->desc.getGCData()->getLiterals()[0].unwrap(); // Strings and numbers are not valid prototypes, so they appear as null. // Externref nulls are also converted to nullref. auto protoType = proto.type.getHeapType(); @@ -3161,4 +3167,79 @@ Literal Literal::getJSPrototype() const { return Literal::makeNull(HeapType::none); } +size_t Literal::getNumElements() const { + assert(isData()); + if (gcData->isRawBytes()) { + auto field = type.getHeapType().getArray().element; + return gcData->getRawBytes().size() / field.getByteSize(); + } + return gcData->getLiterals().size(); +} + +namespace { + +void writeField(void* dest, const Field& field, Literal value) { + if (field.isPacked()) { + assert(field.type == Type::i32); + int32_t c = value.geti32(); + if (field.packedType == Field::i8) { + Bits::writeLE(static_cast(c), dest); + } else if (field.packedType == Field::i16) { + Bits::writeLE(static_cast(c), dest); + } else { + WASM_UNREACHABLE("invalid packed type"); + } + return; + } + + uint8_t buf[16]; + assert(field.getByteSize() <= sizeof(buf)); + value.getBits(buf); + memcpy(dest, buf, field.getByteSize()); +} + +Literal readField(const void* src, const Field& field, bool signed_) { + if (field.isPacked()) { + assert(field.type == Type::i32); + if (field.packedType == Field::i8) { + int8_t val = Bits::readLE(src); + return Literal(signed_ ? int32_t(val) + : int32_t(static_cast(val))); + } else if (field.packedType == Field::i16) { + int16_t val = Bits::readLE(src); + return Literal(signed_ ? int32_t(val) + : int32_t(static_cast(val))); + } else { + WASM_UNREACHABLE("invalid packed type"); + } + } + + return Literal::makeFromMemory(src, field.type); +} + +} // anonymous namespace + +Literal Literal::getElement(size_t index, bool signed_) const { + assert(isData()); + if (gcData->isRawBytes()) { + auto field = type.getHeapType().getArray().element; + size_t elemSize = field.getByteSize(); + assert((index + 1) * elemSize <= gcData->getRawBytes().size()); + return readField(&gcData->getRawBytes()[index * elemSize], field, signed_); + } + return gcData->getLiterals()[index]; +} + +void Literal::setElement(size_t index, Literal value) { + assert(isData()); + if (gcData->isRawBytes()) { + auto field = type.getHeapType().getArray().element; + size_t elemSize = field.getByteSize(); + assert((index + 1) * elemSize <= gcData->getRawBytes().size()); + writeField(&gcData->getRawBytes()[index * elemSize], field, value); + return; + } + gcData->getLiterals()[index] = value; +} + } // namespace wasm