From 8668d71b6cd18d95ab5ae4a354e989a55b50aa02 Mon Sep 17 00:00:00 2001 From: Volker Berlin Date: Sun, 24 Nov 2019 14:44:56 +0100 Subject: [PATCH] Fix the memory offset of strings with a dynamic synthetic function. --- .../jwebassembly/module/FunctionManager.java | 28 ++++-- .../jwebassembly/module/FunctionName.java | 3 +- .../jwebassembly/module/ModuleGenerator.java | 59 ++++++------ .../jwebassembly/module/StringManager.java | 89 ++++++++++++++----- .../jwebassembly/module/WasmCodeBuilder.java | 2 +- .../module/WatCodeSyntheticFunctionName.java | 16 +++- .../runtime/StringOperations.java | 11 +++ 7 files changed, 143 insertions(+), 65 deletions(-) diff --git a/src/de/inetsoftware/jwebassembly/module/FunctionManager.java b/src/de/inetsoftware/jwebassembly/module/FunctionManager.java index 1f500c6..cffea66 100644 --- a/src/de/inetsoftware/jwebassembly/module/FunctionManager.java +++ b/src/de/inetsoftware/jwebassembly/module/FunctionManager.java @@ -20,6 +20,7 @@ import java.util.LinkedHashMap; import java.util.Map; import java.util.Map.Entry; import java.util.function.Function; +import java.util.function.Predicate; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -184,21 +185,21 @@ public class FunctionManager { } /** - * Get the first FunctionName that is required but was not written. + * Get all FunctionName that is required but was not written. * * @return the FunctionName or null */ @Nullable - FunctionName nextWriteLater() { - for( Entry entry : states.entrySet() ) { + Iterator getWriteLater() { + return iterator( entry -> { switch( entry.getValue().state ) { case Needed: case Scanned: - return entry.getKey(); + return true; default: + return false; } - } - return null; + } ); } /** @@ -207,16 +208,25 @@ public class FunctionManager { * @return an iterator */ Iterator getNeededFunctions() { - return states.entrySet().stream().filter( entry -> { + return iterator( entry -> { FunctionState state = entry.getValue(); switch( state.state ) { case Needed: case Scanned: return true; default: + return false; } - return false; - } ).map( entry -> entry.getKey() ).iterator(); + } ); + } + + /** + * get a iterator for function names + * @param filter the filter + * @return the iterator + */ + private Iterator iterator( Predicate> filter ) { + return states.entrySet().stream().filter( filter ).map( entry -> entry.getKey() ).iterator(); } /** diff --git a/src/de/inetsoftware/jwebassembly/module/FunctionName.java b/src/de/inetsoftware/jwebassembly/module/FunctionName.java index 3ff83e0..f7f51d9 100644 --- a/src/de/inetsoftware/jwebassembly/module/FunctionName.java +++ b/src/de/inetsoftware/jwebassembly/module/FunctionName.java @@ -137,7 +137,8 @@ public class FunctionName { if( obj == null ) { return false; } - if( getClass() != obj.getClass() ) { + // synthetic functions should be replace/equals to real functions. + if( !(obj instanceof FunctionName) ) { return false; } FunctionName other = (FunctionName)obj; diff --git a/src/de/inetsoftware/jwebassembly/module/ModuleGenerator.java b/src/de/inetsoftware/jwebassembly/module/ModuleGenerator.java index 5ad17dd..349861d 100644 --- a/src/de/inetsoftware/jwebassembly/module/ModuleGenerator.java +++ b/src/de/inetsoftware/jwebassembly/module/ModuleGenerator.java @@ -292,41 +292,40 @@ public class ModuleGenerator { * if any I/O error occur */ public void finish() throws IOException { - FunctionName next; - while( (next = functions.nextWriteLater()) != null ) { - ClassFile classFile = ClassFile.get( next.className, libraries ); - if( classFile == null ) { - if( next instanceof SyntheticFunctionName ) { - writeMethodImpl( next, true, ((SyntheticFunctionName)next).getCodeBuilder( watParser ) ); - } else { - throw new WasmException( "Missing function: " + next.signatureName, -1 ); - } + for( Iterator it = functions.getWriteLater(); it.hasNext(); ) { + FunctionName next = it.next(); + if( next instanceof SyntheticFunctionName ) { + writeMethodImpl( next, true, ((SyntheticFunctionName)next).getCodeBuilder( watParser ) ); } else { - iterateMethods( classFile, method -> { - try { - FunctionName name; - Map wat = method.getAnnotation( JWebAssembly.TEXTCODE_ANNOTATION ); - if( wat != null ) { - String signature = (String)wat.get( "signature" ); - if( signature == null ) { - signature = method.getType(); + ClassFile classFile = ClassFile.get( next.className, libraries ); + if( classFile == null ) { + throw new WasmException( "Missing function: " + next.signatureName, -1 ); + } else { + MethodInfo method = classFile.getMethod( next.methodName, next.signature ); + if( method != null ) { + try { + Map wat = method.getAnnotation( JWebAssembly.TEXTCODE_ANNOTATION ); + if( wat != null ) { + String signature = (String)wat.get( "signature" ); + if( signature == null ) { + signature = method.getType(); + } + next = new FunctionName( method, signature ); + } else { + method = functions.replace( next, method ); } - name = new FunctionName( method, signature ); - } else { - name = new FunctionName( method ); - method = functions.replace( name, method ); + if( functions.needToWrite( next ) ) { + writeMethod( next, method ); + } + } catch (IOException ioex){ + throw WasmException.create( ioex, sourceFile, className, -1 ); } - if( functions.needToWrite( name ) ) { - writeMethod( name, method ); + } else { + if( functions.needToWrite( next ) ) { + throw new WasmException( "Missing function: " + next.signatureName, -1 ); } - } catch (IOException ioex){ - throw WasmException.create( ioex, sourceFile, className, -1 ); } - } ); - } - - if( functions.needToWrite( next ) ) { - throw new WasmException( "Missing function: " + next.signatureName, -1 ); + } } } javaScript.finish(); diff --git a/src/de/inetsoftware/jwebassembly/module/StringManager.java b/src/de/inetsoftware/jwebassembly/module/StringManager.java index 440dcb0..3996db3 100644 --- a/src/de/inetsoftware/jwebassembly/module/StringManager.java +++ b/src/de/inetsoftware/jwebassembly/module/StringManager.java @@ -23,6 +23,7 @@ import java.nio.charset.StandardCharsets; import java.util.LinkedHashMap; import javax.annotation.Nonnegative; +import javax.annotation.Nonnull; import de.inetsoftware.jwebassembly.api.annotation.WasmTextCode; import de.inetsoftware.jwebassembly.wasm.ValueType; @@ -34,12 +35,51 @@ import de.inetsoftware.jwebassembly.wasm.ValueType; */ class StringManager extends LinkedHashMap { - private FunctionManager functions; + /** + * Signature of method stringConstant. + * + * @see #stringConstant(int) + */ + private static final FunctionName STRING_CONSTANT_FUNCTION = + new FunctionName( "de/inetsoftware/jwebassembly/module/StringManager.stringConstant(I)Ljava/lang/String;" ); + private FunctionManager functions; + + private int stringMemoryOffset = -1; + + /** + * Initialize the string manager. + * + * @param functions + * the function manager + */ void init( FunctionManager functions ) { this.functions = functions; } + /** + * Get the function name object for the {@link #stringConstant(int)}. + * + * @see #stringConstant(int) + * @return the name + */ + @Nonnull + FunctionName getStringConstantFunction() { + if( stringMemoryOffset < 0 ) { + // register the function stringsMemoryOffset() as synthetic function + stringMemoryOffset = 0; + FunctionName offsetFunction = + new WatCodeSyntheticFunctionName( "de/inetsoftware/jwebassembly/module/StringManager", "stringsMemoryOffset", "()I", "", null, ValueType.i32 ) { + protected String getCode() { + return "i32.const " + stringMemoryOffset; + } + }; + functions.markAsNeeded( offsetFunction, true ); + } + + return STRING_CONSTANT_FUNCTION; + } + /** * Finish the prepare. Now no new strings should be added. * @@ -82,10 +122,9 @@ class StringManager extends LinkedHashMap { ByteArrayOutputStream stringOut = new ByteArrayOutputStream(); ByteArrayOutputStream dataStream = writer.dataStream; - int offset = dataStream.size(); - WatCodeSyntheticFunctionName stringMemoryOffset = new WatCodeSyntheticFunctionName( "de/inetsoftware/jwebassembly/module/StringManager", "stringsMemoryOffset", "()I", "i32.const " + offset, null, ValueType.i32 ); -// functions.markAsNeeded( stringMemoryOffset, true ); - offset += size * 4; + // save the offset of the string data for later code inlining + stringMemoryOffset = dataStream.size(); + int offset = stringMemoryOffset + size * 4; for( String str : this.keySet() ) { // write the position where the string starts in the data section @@ -129,18 +168,13 @@ class StringManager extends LinkedHashMap { } /** - * Signature of method stringConstant. - * @see #stringConstant(int) - */ - static final String STRING_CONSTANT_SIGNATURE = "de/inetsoftware/jwebassembly/module/StringManager.stringConstant(I)Ljava/lang/String;"; - - /** - * WASM code - * Get a constant string from the table. + * WASM code

+ * Get a constant string from the table. * - * @param strIdx the id/index of the string. + * @param strIdx + * the id/index of the string. * @return the string - * @see #STRING_CONSTANT_SIGNATURE + * @see #STRING_CONSTANT_FUNCTION */ private static String stringConstant( int strIdx ) { String str = getStringFromTable( strIdx ); @@ -148,25 +182,30 @@ class StringManager extends LinkedHashMap { return str; } + // read the compact string length int offset = getIntFromMemory( strIdx * 4 + stringsMemoryOffset() ); int length = 0; int b; + int shift = 0; do { b = getUnsignedByteFromMemory( offset++ ); - length = (length << 7) + (b & 0x7F); + length += (b & 0x7F) << shift; + shift += 7; } while( b >= 0x80 ); + // copy the bytes from the data section byte[] bytes = new byte[length]; for( int i = 0; i < length; i++ ) { bytes[i] = getUnsignedByteFromMemory( i + offset ); } str = new String( bytes ); + // save the string for future use setStringIntoTable( strIdx, str ); return str; } /** - * WASM code + * WASM code

* Get a string from the string table. Should be inlined from the optimizer. * * @param strIdx @@ -179,7 +218,7 @@ class StringManager extends LinkedHashMap { private static native String getStringFromTable( int strIdx ); /** - * WASM code + * WASM code

* Set a string from the string table. Should be inlined from the optimizer. * * @param strIdx @@ -193,12 +232,18 @@ class StringManager extends LinkedHashMap { "return" ) private static native void setStringIntoTable( int strIdx, String str ); + /** + * WASM code

+ * Placeholder for a synthetic function. Should be inlined from the optimizer. + * @return the memory offset of the string data in the element section + */ + //TODO the annotation can be removed if ModuleGenerator.prepareFunctions() can detect Synthetic functions correctly @WasmTextCode( "i32.const 0" ) private static native int stringsMemoryOffset(); /** - * WASM code - * Load an i32 from memory. The offset must be aligned. + * WASM code

+ * Load an i32 from memory. The offset must be aligned. Should be inlined from the optimizer. * * @param pos * the memory position @@ -210,8 +255,8 @@ class StringManager extends LinkedHashMap { private static native int getIntFromMemory( int pos ); /** - * WASM code - * Load a byte from the memory. + * WASM code

+ * Load a byte from the memory. Should be inlined from the optimizer. * * @param pos * the memory position diff --git a/src/de/inetsoftware/jwebassembly/module/WasmCodeBuilder.java b/src/de/inetsoftware/jwebassembly/module/WasmCodeBuilder.java index 23f72d7..3e4a65e 100644 --- a/src/de/inetsoftware/jwebassembly/module/WasmCodeBuilder.java +++ b/src/de/inetsoftware/jwebassembly/module/WasmCodeBuilder.java @@ -357,7 +357,7 @@ public abstract class WasmCodeBuilder { if( id == null ) { strings.put( (String)value, id = strings.size() ); } - FunctionName name = new FunctionName( StringManager.STRING_CONSTANT_SIGNATURE ); + FunctionName name = strings.getStringConstantFunction(); instructions.add( new WasmConstInstruction( id, ValueType.i32, javaCodePos, lineNumber ) ); addCallInstruction( name, javaCodePos, lineNumber ); } else { diff --git a/src/de/inetsoftware/jwebassembly/module/WatCodeSyntheticFunctionName.java b/src/de/inetsoftware/jwebassembly/module/WatCodeSyntheticFunctionName.java index 7e2f47e..91bf793 100644 --- a/src/de/inetsoftware/jwebassembly/module/WatCodeSyntheticFunctionName.java +++ b/src/de/inetsoftware/jwebassembly/module/WatCodeSyntheticFunctionName.java @@ -16,6 +16,8 @@ */ package de.inetsoftware.jwebassembly.module; +import javax.annotation.Nonnull; + import de.inetsoftware.jwebassembly.wasm.AnyType; import de.inetsoftware.jwebassembly.watparser.WatParser; @@ -57,11 +59,21 @@ class WatCodeSyntheticFunctionName extends ArraySyntheticFunctionName { * @param signatureTypes * the method signature, first the parameters, then null and the the return types */ - public WatCodeSyntheticFunctionName( String className, String name, String signature, String code, AnyType... signatureTypes ) { + public WatCodeSyntheticFunctionName( String className, String name, String signature, @Nonnull String code, AnyType... signatureTypes ) { super( className, name, signature, signatureTypes ); this.code = code; } + /** + * Get Wat code, can be overridden. + * + * @return the code + */ + @Nonnull + protected String getCode() { + return code; + } + /** * {@inheritDoc} */ @@ -75,7 +87,7 @@ class WatCodeSyntheticFunctionName extends ArraySyntheticFunctionName { */ @Override protected WasmCodeBuilder getCodeBuilder( WatParser watParser ) { - watParser.parse( code, null, -1 ); + watParser.parse( getCode(), null, -1 ); return watParser; } } diff --git a/test/de/inetsoftware/jwebassembly/runtime/StringOperations.java b/test/de/inetsoftware/jwebassembly/runtime/StringOperations.java index a5acb67..00e0bba 100644 --- a/test/de/inetsoftware/jwebassembly/runtime/StringOperations.java +++ b/test/de/inetsoftware/jwebassembly/runtime/StringOperations.java @@ -42,6 +42,7 @@ public class StringOperations extends AbstractBaseTest { for( ScriptEngine script : ScriptEngine.testEngines() ) { addParam( list, script, "newFromChars" ); addParam( list, script, "newFromBytes" ); + addParam( list, script, "constant" ); } rule.setTestParameters( list ); return list; @@ -60,5 +61,15 @@ public class StringOperations extends AbstractBaseTest { byte[] bytes = {(byte)0xC3,(byte)0xA4}; return new String( bytes ); } + + @Export + static String constant() { + // string larger as 128 bytes + String constant = "1234567890 äöüäöüß " // umlaute + + "𝟘𝟙𝟚𝟛𝟜𝟝𝟞𝟟𝟠𝟡 𝒥𝒶𝓋𝒶𝓈𝒸𝓇𝒾𝓅𝓉 " // surrogate chars + + "abcdefghijklmnopqrstuvwxyz"; + return constant; + } + } }