[GR-52711] Implement GraalWasm SIMD proposal using the Vector API.

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/truffle/KnownTruffleTypes.java

@@ -69,6 +69,7 @@ public class KnownTruffleTypes extends AbstractKnownTruffleTypes {
     public final ResolvedJavaType BufferOverflowException = lookupType(BufferOverflowException.class);
     public final ResolvedJavaType ReadOnlyBufferException = lookupType(ReadOnlyBufferException.class);
     public final ResolvedJavaType ScopedMemoryAccess_ScopedAccessError = lookupTypeOptional("jdk.internal.misc.ScopedMemoryAccess$ScopedAccessError");
+    public final ResolvedJavaType AssertionError = lookupType(AssertionError.class);
     public final ResolvedJavaType AbstractMemorySegmentImpl = lookupTypeOptional("jdk.internal.foreign.AbstractMemorySegmentImpl");
     public final ResolvedJavaType MemorySegmentProxy = lookupTypeOptional("jdk.internal.access.foreign.MemorySegmentProxy");
 
@@ -245,6 +246,7 @@ private ResolvedJavaType[] createSkippedExceptionTypes() {
         types.add(BufferUnderflowException);
         types.add(BufferOverflowException);
         types.add(ReadOnlyBufferException);
+        types.add(AssertionError);
         return types.toArray(ResolvedJavaType[]::new);
     }
 

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/deopt/DeoptState.java

@@ -186,8 +186,9 @@ private Object materializeObject(int virtualObjectId, FrameInfoQueryResult sourc
 
         if (ImageSingletons.contains(VectorAPIDeoptimizationSupport.class)) {
             VectorAPIDeoptimizationSupport deoptSupport = ImageSingletons.lookup(VectorAPIDeoptimizationSupport.class);
-            Object payloadArray = deoptSupport.materializePayload(this, hub, encodings[curIdx], sourceFrame);
-            if (payloadArray != null) {
+            VectorAPIDeoptimizationSupport.PayloadLayout payloadLayout = deoptSupport.getLayout(DynamicHub.toClass(hub));
+            if (payloadLayout != null) {
+                Object payloadArray = deoptSupport.materializePayload(this, payloadLayout, encodings[curIdx], sourceFrame);
                 JavaConstant arrayConstant = SubstrateObjectConstant.forObject(payloadArray, ReferenceAccess.singleton().haveCompressedReferences());
                 Deoptimizer.writeValueInMaterializedObj(obj, curOffset, arrayConstant, sourceFrame);
                 return obj;

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/deopt/VectorAPIDeoptimizationSupport.java

@@ -33,7 +33,6 @@
 import com.oracle.svm.core.code.FrameInfoQueryResult;
 import com.oracle.svm.core.config.ConfigurationValues;
 import com.oracle.svm.core.config.ObjectLayout;
-import com.oracle.svm.core.hub.DynamicHub;
 
 import jdk.graal.compiler.word.Word;
 import jdk.vm.ci.meta.JavaConstant;
@@ -45,41 +44,36 @@
 public class VectorAPIDeoptimizationSupport {
 
     /**
-     * If the {@code hub} refers to a Vector API vector, materialize its payload array. That is,
-     * allocate a primitive array of the appropriate element type and length for the Vector API
-     * value. Read the vector's entries from the stack and store them in the array.
+     * Materialize the payload array of a Vector API class. That is, allocate a primitive array of
+     * the appropriate element type and length for the Vector API value. Read the vector's entries
+     * from the stack and store them in the array.
      *
      * @param deoptState state for accessing values on the stack
-     * @param hub the hub of the object to be materialized
+     * @param layout non-null payload layout from {@link #getLayout}
      * @param vectorEncoding describes the location of the vector on the stack
      * @param sourceFrame the source frame containing the vector
      * @return a materialized primitive array if the object to be materialized is a Vector API
      *         vector; {@code null} otherwise
      */
-    public Object materializePayload(DeoptState deoptState, DynamicHub hub, FrameInfoQueryResult.ValueInfo vectorEncoding, FrameInfoQueryResult sourceFrame) {
-        Class<?> vectorClass = DynamicHub.toClass(hub);
-        PayloadLayout layout = typeMap.get(vectorClass);
-        if (layout != null) {
-            /*
-             * Read values from the stack and write them to an array of the same element type. Note
-             * that vector masks in states are already represented as vectors of byte-sized 0 or 1
-             * values, this is ensured by the VectorAPIExpansionPhase. Therefore, this code does not
-             * need to worry about the target's representation of vector masks; an element type of
-             * boolean in the layout will allow us to handle masks correctly.
-             */
-            JavaKind elementKind = JavaKind.fromJavaClass(layout.elementType);
-            Object array = Array.newInstance(layout.elementType, layout.vectorLength);
-            ObjectLayout objectLayout = ConfigurationValues.getObjectLayout();
-            UnsignedWord curOffset = Word.unsigned(objectLayout.getArrayBaseOffset(elementKind));
-            for (int i = 0; i < layout.vectorLength; i++) {
-                FrameInfoQueryResult.ValueInfo elementEncoding = vectorEncoding.copyForElement(elementKind, i * elementKind.getByteCount());
-                JavaConstant con = readValue(deoptState, elementEncoding, sourceFrame);
-                writeValueInMaterializedObj(array, curOffset, con, sourceFrame);
-                curOffset = curOffset.add(objectLayout.sizeInBytes(elementKind));
-            }
-            return array;
+    public Object materializePayload(DeoptState deoptState, PayloadLayout layout, FrameInfoQueryResult.ValueInfo vectorEncoding, FrameInfoQueryResult sourceFrame) {
+        /*
+         * Read values from the stack and write them to an array of the same element type. Note that
+         * vector masks in states are already represented as vectors of byte-sized 0 or 1 values,
+         * this is ensured by the VectorAPIExpansionPhase. Therefore, this code does not need to
+         * worry about the target's representation of vector masks; an element type of boolean in
+         * the layout will allow us to handle masks correctly.
+         */
+        JavaKind elementKind = JavaKind.fromJavaClass(layout.elementType);
+        Object array = Array.newInstance(layout.elementType, layout.vectorLength);
+        ObjectLayout objectLayout = ConfigurationValues.getObjectLayout();
+        UnsignedWord curOffset = Word.unsigned(objectLayout.getArrayBaseOffset(elementKind));
+        for (int i = 0; i < layout.vectorLength; i++) {
+            FrameInfoQueryResult.ValueInfo elementEncoding = vectorEncoding.copyForElement(elementKind, i * elementKind.getByteCount());
+            JavaConstant con = readValue(deoptState, elementEncoding, sourceFrame);
+            writeValueInMaterializedObj(array, curOffset, con, sourceFrame);
+            curOffset = curOffset.add(objectLayout.sizeInBytes(elementKind));
         }
-        return null;
+        return array;
     }
 
     protected static JavaConstant readValue(DeoptState deoptState, FrameInfoQueryResult.ValueInfo valueInfo, FrameInfoQueryResult sourceFrame) {
@@ -108,4 +102,8 @@ public record PayloadLayout(Class<?> elementType, int vectorLength) {
     public void putLayout(Class<?> vectorClass, PayloadLayout layout) {
         typeMap.put(vectorClass, layout);
     }
+
+    public PayloadLayout getLayout(Class<?> vectorClass) {
+        return typeMap.get(vectorClass);
+    }
 }

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Target_jdk_internal_vm_vector_VectorSupport.java

@@ -24,6 +24,9 @@
  */
 package com.oracle.svm.core.jdk;
 
+import java.lang.foreign.MemorySegment;
+import java.lang.foreign.ValueLayout;
+import java.util.function.IntFunction;
 import java.util.stream.Collectors;
 
 import org.graalvm.nativeimage.ImageSingletons;
@@ -109,6 +112,48 @@ private static final class Target_jdk_incubator_vector_VectorOperators_Conversio
         @AlwaysInline("Vector API performance")
         private static native Target_jdk_incubator_vector_VectorOperators_ConversionImpl<?, ?> ofReinterpret(Target_jdk_incubator_vector_LaneType dom, Target_jdk_incubator_vector_LaneType ran);
     }
+
+    @TargetClass(className = "jdk.incubator.vector.VectorOperators", innerClass = "Operator", onlyWith = VectorAPIEnabled.class)
+    interface Target_jdk_incubator_vector_VectorOperators_Operator {
+    }
+
+    @TargetClass(className = "jdk.incubator.vector.VectorOperators", innerClass = "ImplCache", onlyWith = VectorAPIEnabled.class)
+    static final class Target_jdk_incubator_vector_VectorOperators_ImplCache<OP extends Target_jdk_incubator_vector_VectorOperators_Operator, T> {
+
+        @Alias Object[] cache;
+
+        /*
+         * We substitute ImplCache#find to remove the call to isNonCapturingLambda. In the process,
+         * we simplify the cache lookup by removing lazy cache initialization as we precompute the
+         * cache.
+         */
+        @Substitute
+        @AlwaysInline("Vector API fast-path")
+        @SuppressWarnings({"unchecked", "unused"})
+        public T find(OP op, int opc, IntFunction<T> supplier) {
+            T fn = (T) cache[opc];
+            return fn;
+        }
+    }
+}
+
+@TargetClass(className = "jdk.incubator.vector.AbstractVector", onlyWith = VectorAPIEnabled.class)
+final class Target_jdk_incubator_vector_AbstractVector {
+}
+
+@TargetClass(className = "jdk.incubator.vector.AbstractSpecies", onlyWith = VectorAPIEnabled.class)
+final class Target_jdk_incubator_vector_AbstractSpecies {
+
+    @Alias private Target_jdk_incubator_vector_AbstractVector dummyVector;
+
+    /*
+     * We initialize the `dummyVector` fields during image build-time using VectorAPIFeature. We can
+     * have the getter method return the precomputed dummy vector directly.
+     */
+    @Substitute
+    Target_jdk_incubator_vector_AbstractVector dummyVector() {
+        return dummyVector;
+    }
 }
 
 @TargetClass(className = "jdk.incubator.vector.ByteVector", onlyWith = VectorAPIEnabled.class)
@@ -119,6 +164,20 @@ final class Target_jdk_incubator_vector_ByteVector {
     @Alias @RecomputeFieldValue(kind = RecomputeFieldValue.Kind.ArrayBaseOffset, declClass = byte[].class, isFinal = true) //
     @TargetElement(name = "ARRAY_BASE") //
     private static long arrayBase;
+
+    @Alias @RecomputeFieldValue(isFinal = true, kind = RecomputeFieldValue.Kind.None) //
+    @TargetElement(name = "ELEMENT_LAYOUT") //
+    static ValueLayout.OfByte elementLayout;
+
+    @Substitute
+    static void memorySegmentSet(MemorySegment ms, long o, int i, byte e) {
+        elementLayout.varHandle().set(ms, o + i * 1L, e);
+    }
+
+    @Substitute
+    static byte memorySegmentGet(MemorySegment ms, long o, int i) {
+        return (byte) elementLayout.varHandle().get(ms, o + i * 1L);
+    }
 }
 
 @TargetClass(className = "jdk.incubator.vector.ShortVector", onlyWith = VectorAPIEnabled.class)
@@ -129,6 +188,20 @@ final class Target_jdk_incubator_vector_ShortVector {
     @Alias @RecomputeFieldValue(kind = RecomputeFieldValue.Kind.ArrayBaseOffset, declClass = short[].class, isFinal = true) //
     @TargetElement(name = "ARRAY_BASE") //
     private static long arrayBase;
+
+    @Alias @RecomputeFieldValue(isFinal = true, kind = RecomputeFieldValue.Kind.None) //
+    @TargetElement(name = "ELEMENT_LAYOUT") //
+    static ValueLayout.OfShort elementLayout;
+
+    @Substitute
+    static void memorySegmentSet(MemorySegment ms, long o, int i, short e) {
+        elementLayout.varHandle().set(ms, o + i * 2L, e);
+    }
+
+    @Substitute
+    static short memorySegmentGet(MemorySegment ms, long o, int i) {
+        return (short) elementLayout.varHandle().get(ms, o + i * 2L);
+    }
 }
 
 @TargetClass(className = "jdk.incubator.vector.IntVector", onlyWith = VectorAPIEnabled.class)
@@ -139,6 +212,20 @@ final class Target_jdk_incubator_vector_IntVector {
     @Alias @RecomputeFieldValue(kind = RecomputeFieldValue.Kind.ArrayBaseOffset, declClass = int[].class, isFinal = true) //
     @TargetElement(name = "ARRAY_BASE") //
     private static long arrayBase;
+
+    @Alias @RecomputeFieldValue(isFinal = true, kind = RecomputeFieldValue.Kind.None) //
+    @TargetElement(name = "ELEMENT_LAYOUT") //
+    static ValueLayout.OfInt elementLayout;
+
+    @Substitute
+    static void memorySegmentSet(MemorySegment ms, long o, int i, int e) {
+        elementLayout.varHandle().set(ms, o + i * 4L, e);
+    }
+
+    @Substitute
+    static int memorySegmentGet(MemorySegment ms, long o, int i) {
+        return (int) elementLayout.varHandle().get(ms, o + i * 4L);
+    }
 }
 
 @TargetClass(className = "jdk.incubator.vector.LongVector", onlyWith = VectorAPIEnabled.class)
@@ -149,6 +236,20 @@ final class Target_jdk_incubator_vector_LongVector {
     @Alias @RecomputeFieldValue(kind = RecomputeFieldValue.Kind.ArrayBaseOffset, declClass = long[].class, isFinal = true) //
     @TargetElement(name = "ARRAY_BASE") //
     private static long arrayBase;
+
+    @Alias @RecomputeFieldValue(isFinal = true, kind = RecomputeFieldValue.Kind.None) //
+    @TargetElement(name = "ELEMENT_LAYOUT") //
+    static ValueLayout.OfLong elementLayout;
+
+    @Substitute
+    static void memorySegmentSet(MemorySegment ms, long o, int i, long e) {
+        elementLayout.varHandle().set(ms, o + i * 8L, e);
+    }
+
+    @Substitute
+    static long memorySegmentGet(MemorySegment ms, long o, int i) {
+        return (long) elementLayout.varHandle().get(ms, o + i * 8L);
+    }
 }
 
 @TargetClass(className = "jdk.incubator.vector.FloatVector", onlyWith = VectorAPIEnabled.class)
@@ -159,6 +260,20 @@ final class Target_jdk_incubator_vector_FloatVector {
     @Alias @RecomputeFieldValue(kind = RecomputeFieldValue.Kind.ArrayBaseOffset, declClass = float[].class, isFinal = true) //
     @TargetElement(name = "ARRAY_BASE") //
     private static long arrayBase;
+
+    @Alias @RecomputeFieldValue(isFinal = true, kind = RecomputeFieldValue.Kind.None) //
+    @TargetElement(name = "ELEMENT_LAYOUT") //
+    static ValueLayout.OfFloat elementLayout;
+
+    @Substitute
+    static void memorySegmentSet(MemorySegment ms, long o, int i, float e) {
+        elementLayout.varHandle().set(ms, o + i * 4L, e);
+    }
+
+    @Substitute
+    static float memorySegmentGet(MemorySegment ms, long o, int i) {
+        return (float) elementLayout.varHandle().get(ms, o + i * 4L);
+    }
 }
 
 @TargetClass(className = "jdk.incubator.vector.DoubleVector", onlyWith = VectorAPIEnabled.class)
@@ -169,4 +284,18 @@ final class Target_jdk_incubator_vector_DoubleVector {
     @Alias @RecomputeFieldValue(kind = RecomputeFieldValue.Kind.ArrayBaseOffset, declClass = double[].class, isFinal = true) //
     @TargetElement(name = "ARRAY_BASE") //
     private static long arrayBase;
+
+    @Alias @RecomputeFieldValue(isFinal = true, kind = RecomputeFieldValue.Kind.None) //
+    @TargetElement(name = "ELEMENT_LAYOUT") //
+    static ValueLayout.OfDouble elementLayout;
+
+    @Substitute
+    static void memorySegmentSet(MemorySegment ms, long o, int i, double e) {
+        elementLayout.varHandle().set(ms, o + i * 8L, e);
+    }
+
+    @Substitute
+    static double memorySegmentGet(MemorySegment ms, long o, int i) {
+        return (double) elementLayout.varHandle().get(ms, o + i * 8L);
+    }
 }

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/methodhandles/Target_java_lang_invoke_MethodHandle.java

@@ -285,8 +285,11 @@ private static SubstrateConstructorAccessor getConstructorAccessor(Constructor<?
     }
 
     private static <T extends AccessibleObject & Member> void checkMember(T member, boolean isStatic) {
-        VMError.guarantee(Modifier.isStatic(member.getModifiers()) == isStatic,
-                        "Cannot perform %s operation on a %s member".formatted(isStatic ? "static" : "non-static", isStatic ? "non-static" : "static"));
+        if (Modifier.isStatic(member.getModifiers()) != isStatic) {
+            throw VMError.shouldNotReachHere("Cannot perform " +
+                            (isStatic ? "static" : "non-static") + " operation on a " +
+                            (isStatic ? "non-static" : "static") + " member");
+        }
     }
 
     private static SubstrateAccessor getAccessor(Target_java_lang_invoke_MemberName memberName) {
@@ -295,7 +298,10 @@ private static SubstrateAccessor getAccessor(Target_java_lang_invoke_MemberName
     }
 
     private static void checkArgs(Object[] args, int expectedLength, String methodName) {
-        VMError.guarantee((expectedLength == 0 && args == null) || args.length == expectedLength, "%s requires exactly %d arguments".formatted(methodName, expectedLength));
+        if ((expectedLength == 0 && args == null) || args.length == expectedLength) {
+            return;
+        }
+        throw VMError.shouldNotReachHere(methodName + " requires exactly " + expectedLength + " arguments");
     }
 
     private static void convertArgs(Object[] args, MethodType methodType) throws Throwable {