From 09d122eaa7d8f8e2d2987f32c03d57ea533105dd Mon Sep 17 00:00:00 2001 From: Kelvin Nishikawa Date: Sat, 16 Nov 2024 23:21:30 -0800 Subject: [PATCH 1/5] Adding Feature.Detect project --- .gitignore | 1 + .gitmodules | 3 + Feature.Detect/Feature.Detect.csproj | 14 +++ Feature.Detect/esm2module.mjs | 129 +++++++++++++++++++++++++++ Feature.Detect/wasm-feature-detect | 1 + WACS.sln | 6 ++ 6 files changed, 154 insertions(+) create mode 100644 Feature.Detect/Feature.Detect.csproj create mode 100644 Feature.Detect/esm2module.mjs create mode 160000 Feature.Detect/wasm-feature-detect diff --git a/.gitignore b/.gitignore index 453a27b..c647b94 100644 --- a/.gitignore +++ b/.gitignore @@ -84,6 +84,7 @@ output/ riderModule.iml /_ReSharper.Caches/ +/Feature.Detect/generated-wasm/** /Spec.Test/generated-json/** /Wacs.Console/Data/** diff --git a/.gitmodules b/.gitmodules index ebc337c..62c73e2 100644 --- a/.gitmodules +++ b/.gitmodules @@ -7,3 +7,6 @@ [submodule "unity"] path = unity url = git@github.com:kelnishi/WACS-Unity.git +[submodule "Feature.Detect/wasm-feature-detect"] + path = Feature.Detect/wasm-feature-detect + url = git@github.com:GoogleChromeLabs/wasm-feature-detect.git diff --git a/Feature.Detect/Feature.Detect.csproj b/Feature.Detect/Feature.Detect.csproj new file mode 100644 index 0000000..a155a49 --- /dev/null +++ b/Feature.Detect/Feature.Detect.csproj @@ -0,0 +1,14 @@ + + + + Exe + net6.0 + enable + enable + + + + + + + diff --git a/Feature.Detect/esm2module.mjs b/Feature.Detect/esm2module.mjs new file mode 100644 index 0000000..5445eb0 --- /dev/null +++ b/Feature.Detect/esm2module.mjs @@ -0,0 +1,129 @@ +import fs from 'fs'; +import path from 'path'; + +// Check if a filename and output directory are provided as arguments +if (process.argv.length < 4) { + console.error('Usage: node createWasm.mjs '); + process.exit(1); +} + +// Get the file path and output directory from the arguments +const indexFilePath = process.argv[2]; +const outputDirectory = process.argv[3]; + +// Extract the output file name using the directory name of the index.js file +const outputFileName = path.basename(path.dirname(indexFilePath)); +const outputFilePath = path.join(outputDirectory, `${outputFileName}.wasm`); +const outputJsonPath = path.join(outputDirectory, `${outputFileName}.json`); + +// Ensure the output directory exists +if (!fs.existsSync(outputDirectory)) { + fs.mkdirSync(outputDirectory, { recursive: true }); +} + +// Save the original WebAssembly functions +const OriginalWebAssemblyModule = WebAssembly.Module; +const OriginalWebAssemblyInstantiate = WebAssembly.instantiate; +const OriginalWebAssemblyValidate = WebAssembly.validate; + +// Helper function to convert various input types to Uint8Array +function toUint8Array(bufferSource) { + if (bufferSource instanceof Uint8Array) { + return bufferSource; + } else if (Array.isArray(bufferSource)) { + return new Uint8Array(bufferSource); + } else if (bufferSource instanceof ArrayBuffer) { + return new Uint8Array(bufferSource); + } else if (bufferSource instanceof SharedArrayBuffer) { + return new Uint8Array(bufferSource); + } else { + throw new TypeError('Expected Uint8Array, ArrayBuffer, or Array'); + } +} + +// Helper function to write the WASM buffer +function writeWasmBuffer(bufferSource, source) { + try { + const uint8Array = toUint8Array(bufferSource); + fs.writeFileSync(outputFilePath, uint8Array); + console.log(`Wrote ${uint8Array.length} bytes to ${outputFilePath} (source: ${source})`); + return uint8Array; + } catch (error) { + console.error('Error writing WASM buffer:', error); + throw error; + } +} + +// Helper function to write metadata +function writeMetadata(metadata) { + try { + const metadataWithBytes = { + ...metadata, + byteLength: metadata.bytes?.length, + bytes: Array.from(metadata.bytes || []), // Convert Uint8Array to regular array for JSON + }; + fs.writeFileSync(outputJsonPath, JSON.stringify(metadataWithBytes, null, 2)); + console.log(`Wrote metadata to ${outputJsonPath}`); + } catch (error) { + console.error('Error writing metadata:', error); + throw error; + } +} + +// Monkey patch WebAssembly.Module +WebAssembly.Module = function(bufferSource) { + console.log('Intercepted WebAssembly.Module call'); + const bytes = writeWasmBuffer(bufferSource, 'Module'); + writeMetadata({ + source: 'Module', + timestamp: new Date().toISOString(), + bytes + }); + return new OriginalWebAssemblyModule(bufferSource); +}; + +// Monkey patch WebAssembly.instantiate +WebAssembly.instantiate = function(bufferSource, importObject, options) { + console.log('Intercepted WebAssembly.instantiate call'); + const bytes = writeWasmBuffer(bufferSource, 'instantiate'); + + const metadata = { + source: 'instantiate', + timestamp: new Date().toISOString(), + importObject: importObject || {}, + options: options || {}, + bytes + }; + + writeMetadata(metadata); + + return OriginalWebAssemblyInstantiate(bufferSource, importObject, options); +}; + +// Monkey patch WebAssembly.validate +WebAssembly.validate = function(bufferSource) { + console.log('Intercepted WebAssembly.validate call'); + const bytes = writeWasmBuffer(bufferSource, 'validate'); + + const metadata = { + source: 'validate', + timestamp: new Date().toISOString(), + bytes + }; + + writeMetadata(metadata); + + return OriginalWebAssemblyValidate(bufferSource); +}; + +// Dynamically import the index file +(async () => { + try { + const module = await import(indexFilePath); + const result = await module.default(); + console.log('Function executed:', result); + } catch (error) { + console.error('Error executing module:', error); + process.exit(1); + } +})(); \ No newline at end of file diff --git a/Feature.Detect/wasm-feature-detect b/Feature.Detect/wasm-feature-detect new file mode 160000 index 0000000..8bfe669 --- /dev/null +++ b/Feature.Detect/wasm-feature-detect @@ -0,0 +1 @@ +Subproject commit 8bfe6691b0749b53d605f3220f15e68751c4b5b6 diff --git a/WACS.sln b/WACS.sln index ea7270b..76e1c31 100644 --- a/WACS.sln +++ b/WACS.sln @@ -8,6 +8,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wacs.WASIp1", "Wacs.WASIp1\ EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spec.Test", "Spec.Test\Spec.Test.csproj", "{2200F588-3506-4A4B-BD86-1C8FABA5A19E}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Feature.Detect", "Feature.Detect\Feature.Detect.csproj", "{AE0D7913-8AA2-4054-B7BC-0C522584272C}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -30,5 +32,9 @@ Global {2200F588-3506-4A4B-BD86-1C8FABA5A19E}.Debug|Any CPU.Build.0 = Debug|Any CPU {2200F588-3506-4A4B-BD86-1C8FABA5A19E}.Release|Any CPU.ActiveCfg = Release|Any CPU {2200F588-3506-4A4B-BD86-1C8FABA5A19E}.Release|Any CPU.Build.0 = Release|Any CPU + {AE0D7913-8AA2-4054-B7BC-0C522584272C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {AE0D7913-8AA2-4054-B7BC-0C522584272C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {AE0D7913-8AA2-4054-B7BC-0C522584272C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {AE0D7913-8AA2-4054-B7BC-0C522584272C}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection EndGlobal From 0f395c49c5a2eca31a1b7a367f455d66291897e1 Mon Sep 17 00:00:00 2001 From: Kelvin Nishikawa Date: Sun, 17 Nov 2024 00:13:09 -0800 Subject: [PATCH 2/5] Feature detect script generates from .wat or .js files. Fixing ignores --- .gitignore | 2 - Feature.Detect/.gitignore | 37 +++++ Feature.Detect/convert_detector.mjs | 217 ++++++++++++++++++++++++++++ Feature.Detect/esm2module.mjs | 129 ----------------- Feature.Detect/package.json | 5 + Spec.Test/.gitignore | 3 + 6 files changed, 262 insertions(+), 131 deletions(-) create mode 100644 Feature.Detect/.gitignore create mode 100644 Feature.Detect/convert_detector.mjs delete mode 100644 Feature.Detect/esm2module.mjs create mode 100644 Feature.Detect/package.json create mode 100644 Spec.Test/.gitignore diff --git a/.gitignore b/.gitignore index c647b94..ada60b1 100644 --- a/.gitignore +++ b/.gitignore @@ -84,8 +84,6 @@ output/ riderModule.iml /_ReSharper.Caches/ -/Feature.Detect/generated-wasm/** -/Spec.Test/generated-json/** /Wacs.Console/Data/** .DS_Store diff --git a/Feature.Detect/.gitignore b/Feature.Detect/.gitignore new file mode 100644 index 0000000..75b7c11 --- /dev/null +++ b/Feature.Detect/.gitignore @@ -0,0 +1,37 @@ +# Node modules +node_modules/ +npm-debug.log +yarn-error.log +package-lock.json +yarn.lock + +# Build outputs +dist/ +build/ + +# Environment variables +.env +.env.local +.env.*.local + +# IDE and editor specific files +.idea/ +.vscode/ +*.sublime-project +*.sublime-workspace + +# OS generated files +.DS_Store +Thumbs.db + +# Logs +logs/ +*.log + +# Temporary files +tmp/ +temp/ + + +#generated files +/generated-wasm/** \ No newline at end of file diff --git a/Feature.Detect/convert_detector.mjs b/Feature.Detect/convert_detector.mjs new file mode 100644 index 0000000..ca29cd5 --- /dev/null +++ b/Feature.Detect/convert_detector.mjs @@ -0,0 +1,217 @@ +import fs from 'fs'; +import path from 'path'; +import wabt from 'wabt'; +import { fileURLToPath } from 'url'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); + +// Check if a directory is provided as argument +if (process.argv.length < 3) { + console.error('Usage: node createWasm.mjs [output_directory]'); + process.exit(1); +} + +// Get the input and output directories from the arguments +const inputDirectory = process.argv[2]; +const outputDirectory = process.argv[3] || inputDirectory; + +const watPath = path.join(inputDirectory, 'module.wat'); +const indexPath = path.join(inputDirectory, 'index.js'); + +const outputFileName = path.basename(inputDirectory); +const outputFilePath = path.join(outputDirectory, `${outputFileName}.wasm`); +const outputJsonPath = path.join(outputDirectory, `${outputFileName}.json`); + +// Ensure the output directory exists +if (!fs.existsSync(outputDirectory)) { + fs.mkdirSync(outputDirectory, { recursive: true }); +} + +// Function to extract proposal information from source file +function extractProposalInfo(content) { + try { + const proposalMatch = content.match(/;; Proposal: (https:\/\/[^\n]+)/); + const nameMatch = content.match(/;; Name: ([^\n]+)/); + const featuresMatch = content.match(/;; Features: ([^\n]+)/); + + return { + name: nameMatch ? nameMatch[1].trim() : undefined, + proposal: proposalMatch ? proposalMatch[1].trim() : undefined, + features: featuresMatch ? featuresMatch[1].trim().split(',').map(f => f.trim()) : undefined + }; + } catch (error) { + console.error('Error extracting proposal info:', error); + return {}; + } +} + +// Function to transform JS code to remove imports and exports +function transformCode(sourceCode) { + // Remove import statements + sourceCode = sourceCode.replace(/import[^;]+;/g, ''); + + // Extract the main function body + const functionMatch = sourceCode.match(/export default async \(\) => {([\s\S]+)}/); + if (functionMatch) { + return `async () => {${functionMatch[1]}}`; + } + + return sourceCode; +} + +// Helper function to convert various input types to Uint8Array +function toUint8Array(bufferSource) { + if (bufferSource instanceof Uint8Array) { + return bufferSource; + } else if (Array.isArray(bufferSource)) { + return new Uint8Array(bufferSource); + } else if (bufferSource instanceof ArrayBuffer) { + return new Uint8Array(bufferSource); + } else if (bufferSource instanceof SharedArrayBuffer) { + return new Uint8Array(bufferSource); + } else { + throw new TypeError('Expected Uint8Array, ArrayBuffer, or Array'); + } +} + +// Function to handle WAT file +async function processWatFile(watPath, outputBasePath) { + console.log('Processing WAT file:', watPath); + try { + // Read and parse the WAT file + const watContent = fs.readFileSync(watPath, 'utf8'); + const proposalInfo = extractProposalInfo(watContent); + + // Initialize wabt and convert WAT to WASM + const wabtInstance = await wabt(); + const wasmModule = wabtInstance.parseWat('module.wat', watContent); + const { buffer } = wasmModule.toBinary({}); + + // Write WASM file + const outputWasmPath = outputBasePath + '.wasm'; + fs.writeFileSync(outputWasmPath, Buffer.from(buffer)); + console.log(`Wrote ${buffer.byteLength} bytes to ${outputWasmPath}`); + + // Write metadata + const metadata = { + source: 'wat2wasm', + timestamp: new Date().toISOString(), + ...proposalInfo, + }; + + const outputJsonPath = outputBasePath + '.json'; + fs.writeFileSync(outputJsonPath, JSON.stringify(metadata, null, 2)); + console.log(`Wrote metadata to ${outputJsonPath}`); + + return true; + } catch (error) { + console.error('Error processing WAT file:', error); + return false; + } +} + +let captured = false; +let currentSourceFile = null; + +// Helper function to write metadata for JS-generated WASM +function writeMetadata(metadata, sourceFile) { + try { + const content = fs.readFileSync(sourceFile, 'utf8'); + const proposalInfo = extractProposalInfo(content); + const metadataWithBytes = { + ...metadata, + ...proposalInfo, + }; + + const outputJsonPath = path.join(outputDirectory, path.basename(path.dirname(sourceFile)) + '.json'); + fs.writeFileSync(outputJsonPath, JSON.stringify(metadataWithBytes, null, 2)); + console.log(`Wrote metadata to ${outputJsonPath}`); + } catch (error) { + console.error('Error writing metadata:', error); + throw error; + } +} + +// Helper function to write the WASM buffer from JS +function writeWasmBuffer(bufferSource, source, sourceFile) { + try { + const uint8Array = toUint8Array(bufferSource); + const outputWasmPath = path.join(outputDirectory, path.basename(path.dirname(sourceFile)) + '.wasm'); + fs.writeFileSync(outputWasmPath, uint8Array); + console.log(`Wrote ${uint8Array.length} bytes to ${outputWasmPath} (source: ${source})`); + captured = true; + return uint8Array; + } catch (error) { + console.error('Error writing WASM buffer:', error); + throw error; + } +} + + +// Monkey patch WebAssembly.Module +WebAssembly.Module = function(bufferSource) { + const bytes = writeWasmBuffer(bufferSource, 'Module', indexPath); + writeMetadata({ + source: 'WebAssembly.Module', + }, indexPath); + + return 'Intercepted WebAssembly.Module call'; +}; + +// Monkey patch WebAssembly.instantiate +WebAssembly.instantiate = function(bufferSource, importObject, options) { + const bytes = writeWasmBuffer(bufferSource, 'instantiate', indexPath); + + const metadata = { + source: 'WebAssembly.instantiate', + options: options || {}, + }; + + writeMetadata(metadata, indexPath); + return 'Intercepted WebAssembly.instantiate call'; +}; + +// Monkey patch WebAssembly.validate +WebAssembly.validate = function(bufferSource) { + const bytes = writeWasmBuffer(bufferSource, 'validate', indexPath); + + const metadata = { + source: 'WebAssembly.validate', + }; + + writeMetadata(metadata, indexPath); + return 'Intercepted WebAssembly.validate call'; +}; + + +// Main function to process directory +async function processDirectory(inputDir) { + const outputBasePath = path.join(outputDirectory, path.basename(inputDir)); + + if (fs.existsSync(watPath)) { + console.log('Found module.wat'); + return processWatFile(watPath, outputBasePath); + } else if (fs.existsSync(indexPath)) { + console.log('Found index.js'); + try { + // Read and transform the source code + const module = await import("./"+indexPath); + const result = await module.default(); + console.log('Function executed:', result); + return true; + } catch (error) { + console.error('Error executing JS code:', error); + return false; + } + } else { + console.error('No module.wat or index.js found in directory'); + return false; + } +} + +// Run the script +processDirectory(inputDirectory).catch(error => { + console.error('Error:', error); + process.exit(1); +}); \ No newline at end of file diff --git a/Feature.Detect/esm2module.mjs b/Feature.Detect/esm2module.mjs deleted file mode 100644 index 5445eb0..0000000 --- a/Feature.Detect/esm2module.mjs +++ /dev/null @@ -1,129 +0,0 @@ -import fs from 'fs'; -import path from 'path'; - -// Check if a filename and output directory are provided as arguments -if (process.argv.length < 4) { - console.error('Usage: node createWasm.mjs '); - process.exit(1); -} - -// Get the file path and output directory from the arguments -const indexFilePath = process.argv[2]; -const outputDirectory = process.argv[3]; - -// Extract the output file name using the directory name of the index.js file -const outputFileName = path.basename(path.dirname(indexFilePath)); -const outputFilePath = path.join(outputDirectory, `${outputFileName}.wasm`); -const outputJsonPath = path.join(outputDirectory, `${outputFileName}.json`); - -// Ensure the output directory exists -if (!fs.existsSync(outputDirectory)) { - fs.mkdirSync(outputDirectory, { recursive: true }); -} - -// Save the original WebAssembly functions -const OriginalWebAssemblyModule = WebAssembly.Module; -const OriginalWebAssemblyInstantiate = WebAssembly.instantiate; -const OriginalWebAssemblyValidate = WebAssembly.validate; - -// Helper function to convert various input types to Uint8Array -function toUint8Array(bufferSource) { - if (bufferSource instanceof Uint8Array) { - return bufferSource; - } else if (Array.isArray(bufferSource)) { - return new Uint8Array(bufferSource); - } else if (bufferSource instanceof ArrayBuffer) { - return new Uint8Array(bufferSource); - } else if (bufferSource instanceof SharedArrayBuffer) { - return new Uint8Array(bufferSource); - } else { - throw new TypeError('Expected Uint8Array, ArrayBuffer, or Array'); - } -} - -// Helper function to write the WASM buffer -function writeWasmBuffer(bufferSource, source) { - try { - const uint8Array = toUint8Array(bufferSource); - fs.writeFileSync(outputFilePath, uint8Array); - console.log(`Wrote ${uint8Array.length} bytes to ${outputFilePath} (source: ${source})`); - return uint8Array; - } catch (error) { - console.error('Error writing WASM buffer:', error); - throw error; - } -} - -// Helper function to write metadata -function writeMetadata(metadata) { - try { - const metadataWithBytes = { - ...metadata, - byteLength: metadata.bytes?.length, - bytes: Array.from(metadata.bytes || []), // Convert Uint8Array to regular array for JSON - }; - fs.writeFileSync(outputJsonPath, JSON.stringify(metadataWithBytes, null, 2)); - console.log(`Wrote metadata to ${outputJsonPath}`); - } catch (error) { - console.error('Error writing metadata:', error); - throw error; - } -} - -// Monkey patch WebAssembly.Module -WebAssembly.Module = function(bufferSource) { - console.log('Intercepted WebAssembly.Module call'); - const bytes = writeWasmBuffer(bufferSource, 'Module'); - writeMetadata({ - source: 'Module', - timestamp: new Date().toISOString(), - bytes - }); - return new OriginalWebAssemblyModule(bufferSource); -}; - -// Monkey patch WebAssembly.instantiate -WebAssembly.instantiate = function(bufferSource, importObject, options) { - console.log('Intercepted WebAssembly.instantiate call'); - const bytes = writeWasmBuffer(bufferSource, 'instantiate'); - - const metadata = { - source: 'instantiate', - timestamp: new Date().toISOString(), - importObject: importObject || {}, - options: options || {}, - bytes - }; - - writeMetadata(metadata); - - return OriginalWebAssemblyInstantiate(bufferSource, importObject, options); -}; - -// Monkey patch WebAssembly.validate -WebAssembly.validate = function(bufferSource) { - console.log('Intercepted WebAssembly.validate call'); - const bytes = writeWasmBuffer(bufferSource, 'validate'); - - const metadata = { - source: 'validate', - timestamp: new Date().toISOString(), - bytes - }; - - writeMetadata(metadata); - - return OriginalWebAssemblyValidate(bufferSource); -}; - -// Dynamically import the index file -(async () => { - try { - const module = await import(indexFilePath); - const result = await module.default(); - console.log('Function executed:', result); - } catch (error) { - console.error('Error executing module:', error); - process.exit(1); - } -})(); \ No newline at end of file diff --git a/Feature.Detect/package.json b/Feature.Detect/package.json new file mode 100644 index 0000000..29db21c --- /dev/null +++ b/Feature.Detect/package.json @@ -0,0 +1,5 @@ +{ + "dependencies": { + "wabt": "^1.0.36" + } +} diff --git a/Spec.Test/.gitignore b/Spec.Test/.gitignore new file mode 100644 index 0000000..fdc4f25 --- /dev/null +++ b/Spec.Test/.gitignore @@ -0,0 +1,3 @@ + +# generated files +/generated-json/** \ No newline at end of file From e49c3f6b476bbd3427da2aebd1c17edda3aeff4e Mon Sep 17 00:00:00 2001 From: Kelvin Nishikawa Date: Sun, 17 Nov 2024 13:02:59 -0800 Subject: [PATCH 3/5] Adding build_feature_detect.sh --- Feature.Detect/build_feature_detect.sh | 18 ++++++++ Feature.Detect/convert_detector.mjs | 61 ++++++++++++++++---------- 2 files changed, 57 insertions(+), 22 deletions(-) create mode 100755 Feature.Detect/build_feature_detect.sh diff --git a/Feature.Detect/build_feature_detect.sh b/Feature.Detect/build_feature_detect.sh new file mode 100755 index 0000000..5171f19 --- /dev/null +++ b/Feature.Detect/build_feature_detect.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +# Set the parent directory +PARENT_DIR="./wasm-feature-detect/src/detectors" + +OUTPUT_DIR="./generated-wasm" + +mkdir -p "$OUTPUT_DIR" + +# Iterate over all directories in the parent directory +for dir in "$PARENT_DIR"/*/; do + # Check if it is indeed a directory + if [ -d "$dir" ]; then + echo "Processing directory: $dir" + # Run the node command with the directory as an argument + node convert_detector.mjs "$dir" "$OUTPUT_DIR" + fi +done diff --git a/Feature.Detect/convert_detector.mjs b/Feature.Detect/convert_detector.mjs index ca29cd5..9f25c31 100644 --- a/Feature.Detect/convert_detector.mjs +++ b/Feature.Detect/convert_detector.mjs @@ -46,20 +46,6 @@ function extractProposalInfo(content) { } } -// Function to transform JS code to remove imports and exports -function transformCode(sourceCode) { - // Remove import statements - sourceCode = sourceCode.replace(/import[^;]+;/g, ''); - - // Extract the main function body - const functionMatch = sourceCode.match(/export default async \(\) => {([\s\S]+)}/); - if (functionMatch) { - return `async () => {${functionMatch[1]}}`; - } - - return sourceCode; -} - // Helper function to convert various input types to Uint8Array function toUint8Array(bufferSource) { if (bufferSource instanceof Uint8Array) { @@ -84,8 +70,13 @@ async function processWatFile(watPath, outputBasePath) { const proposalInfo = extractProposalInfo(watContent); // Initialize wabt and convert WAT to WASM + let features = proposalInfo.features ?? []; const wabtInstance = await wabt(); - const wasmModule = wabtInstance.parseWat('module.wat', watContent); + const wasmModule = wabtInstance.parseWat( + 'module.wat', + watContent, + Object.fromEntries(features.map((flag) => [flag, true])),); + const { buffer } = wasmModule.toBinary({}); // Write WASM file @@ -93,10 +84,13 @@ async function processWatFile(watPath, outputBasePath) { fs.writeFileSync(outputWasmPath, Buffer.from(buffer)); console.log(`Wrote ${buffer.byteLength} bytes to ${outputWasmPath}`); + const outputFilename = path.basename(outputWasmPath); + // Write metadata const metadata = { source: 'wat2wasm', timestamp: new Date().toISOString(), + module: outputFilename, ...proposalInfo, }; @@ -141,30 +135,49 @@ function writeWasmBuffer(bufferSource, source, sourceFile) { fs.writeFileSync(outputWasmPath, uint8Array); console.log(`Wrote ${uint8Array.length} bytes to ${outputWasmPath} (source: ${source})`); captured = true; - return uint8Array; + + const outputFilename = path.basename(outputWasmPath); + return outputFilename; } catch (error) { console.error('Error writing WASM buffer:', error); throw error; } } +const handler = { + has(target, prop) { + // Log the property being checked with "in" + // console.log(`Checking presence of property "${String(prop)}" in WebAssembly`); + writeMetadata({ + source: `"${String(prop)}" in WebAssembly`, + }, indexPath); + + // Return the result of the default behavior + return Reflect.has(target, prop); + } +}; + +// Create a proxy for the WebAssembly object +const webAssemblyProxy = new Proxy(WebAssembly, handler); // Monkey patch WebAssembly.Module -WebAssembly.Module = function(bufferSource) { - const bytes = writeWasmBuffer(bufferSource, 'Module', indexPath); +webAssemblyProxy.Module = function(bufferSource) { + const outfile = writeWasmBuffer(bufferSource, 'Module', indexPath); writeMetadata({ source: 'WebAssembly.Module', + module: outfile, }, indexPath); return 'Intercepted WebAssembly.Module call'; }; // Monkey patch WebAssembly.instantiate -WebAssembly.instantiate = function(bufferSource, importObject, options) { - const bytes = writeWasmBuffer(bufferSource, 'instantiate', indexPath); +webAssemblyProxy.instantiate = function(bufferSource, importObject, options) { + const outfile = writeWasmBuffer(bufferSource, 'instantiate', indexPath); const metadata = { source: 'WebAssembly.instantiate', + module: outfile, options: options || {}, }; @@ -173,17 +186,20 @@ WebAssembly.instantiate = function(bufferSource, importObject, options) { }; // Monkey patch WebAssembly.validate -WebAssembly.validate = function(bufferSource) { - const bytes = writeWasmBuffer(bufferSource, 'validate', indexPath); +webAssemblyProxy.validate = function(bufferSource) { + const outfile = writeWasmBuffer(bufferSource, 'validate', indexPath); const metadata = { source: 'WebAssembly.validate', + module: outfile, }; writeMetadata(metadata, indexPath); return 'Intercepted WebAssembly.validate call'; }; +// Replace the original WebAssembly with the proxy +global.WebAssembly = webAssemblyProxy; // Main function to process directory async function processDirectory(inputDir) { @@ -198,6 +214,7 @@ async function processDirectory(inputDir) { // Read and transform the source code const module = await import("./"+indexPath); const result = await module.default(); + console.log('Function executed:', result); return true; } catch (error) { From 12ff17eeb83099bba3cd8634739f6dfa68634138 Mon Sep 17 00:00:00 2001 From: Kelvin Nishikawa Date: Sun, 17 Nov 2024 14:56:03 -0800 Subject: [PATCH 4/5] Standing up Feature.Detect test runner --- .github/workflows/ci.yml | 2 +- Feature.Detect/DetectFeatures.cs | 43 +++++++++++++ Feature.Detect/Feature.Detect.csproj | 21 +++++++ Feature.Detect/FeatureDetectTestData.cs | 73 +++++++++++++++++++++++ Feature.Detect/FeatureJson/FeatureJson.cs | 54 +++++++++++++++++ Feature.Detect/testsettings.json | 3 + 6 files changed, 195 insertions(+), 1 deletion(-) create mode 100644 Feature.Detect/DetectFeatures.cs create mode 100644 Feature.Detect/FeatureDetectTestData.cs create mode 100644 Feature.Detect/FeatureJson/FeatureJson.cs create mode 100644 Feature.Detect/testsettings.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ecf315b..64b590a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -54,4 +54,4 @@ jobs: run: dotnet build --configuration Release --no-restore - name: Run Tests - run: dotnet test --configuration Release --no-build --verbosity normal \ No newline at end of file + run: dotnet test --configuration Release --no-build --verbosity normal Spec.Test/Spec.Test.csproj \ No newline at end of file diff --git a/Feature.Detect/DetectFeatures.cs b/Feature.Detect/DetectFeatures.cs new file mode 100644 index 0000000..fe8232b --- /dev/null +++ b/Feature.Detect/DetectFeatures.cs @@ -0,0 +1,43 @@ +using Spec.Test; +using Wacs.Core; +using Wacs.Core.Runtime; +using Wacs.Core.Types; +using Xunit; + +namespace Feature.Detect; + +public class DetectFeatures +{ + [Theory] + [ClassData(typeof(FeatureDetectTestData))] + public void Detect(FeatureJson.FeatureJson file) + { + if (!string.IsNullOrEmpty(file.Module)) + { + try + { + var runtime = new WasmRuntime(); + + //Mutable globals + var mutableGlobal = new GlobalType(ValType.I32, Mutability.Mutable); + runtime.BindHostGlobal(("a", "b"), mutableGlobal, 1); + + var filepath = Path.Combine(file.Path, file.Module); + using var fileStream = new FileStream(filepath, FileMode.Open); + var module = BinaryModuleParser.ParseWasm(fileStream); + var modInst = runtime.InstantiateModule(module); + var moduleName = !string.IsNullOrEmpty(file.Name)?file.Name:$"{filepath}"; + module.SetName(moduleName); + } + catch (Exception e) + { + Assert.Fail($"{file.Name} support not detected.\n{e}"); + } + + } + else + { + Assert.Fail($"{file.Name} not supported."); + } + } +} \ No newline at end of file diff --git a/Feature.Detect/Feature.Detect.csproj b/Feature.Detect/Feature.Detect.csproj index a155a49..f91be62 100644 --- a/Feature.Detect/Feature.Detect.csproj +++ b/Feature.Detect/Feature.Detect.csproj @@ -11,4 +11,25 @@ + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + Always + + + + + + + diff --git a/Feature.Detect/FeatureDetectTestData.cs b/Feature.Detect/FeatureDetectTestData.cs new file mode 100644 index 0000000..4387785 --- /dev/null +++ b/Feature.Detect/FeatureDetectTestData.cs @@ -0,0 +1,73 @@ +// /* +// * Copyright 2024 Kelvin Nishikawa +// * +// * Licensed under the Apache License, Version 2.0 (the "License"); +// * you may not use this file except in compliance with the License. +// * You may obtain a copy of the License at +// * +// * http://www.apache.org/licenses/LICENSE-2.0 +// * +// * Unless required by applicable law or agreed to in writing, software +// * distributed under the License is distributed on an "AS IS" BASIS, +// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// * See the License for the specific language governing permissions and +// * limitations under the License. +// */ + +using System.Collections; +using System.Text.Json; +using System.Text.Json.Serialization; +using Feature.Detect.FeatureJson; +using Microsoft.Extensions.Configuration; + +namespace Spec.Test +{ + public class FeatureDetectTestData : IEnumerable + { + private static readonly IConfiguration Configuration; + + static FeatureDetectTestData() + { + // Use ConfigurationFixture to get the JSON directory path + Configuration = new ConfigurationBuilder() + .SetBasePath(AppContext.BaseDirectory) // Set to current directory + .AddJsonFile("testsettings.json", optional: false, reloadOnChange: true) + .Build(); + } + + private static string JsonDirectory => Path.Combine(AppContext.BaseDirectory, Configuration["JsonDirectory"] ?? ""); + + public IEnumerator GetEnumerator() + { + var files = Directory.GetFiles(JsonDirectory, "*.json", SearchOption.AllDirectories).OrderBy(path => path); + foreach (var file in files) + { + var testData = LoadFeatureDefinition(file); + yield return new object[] { testData }; + } + } + + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + + static FeatureJson LoadFeatureDefinition(string jsonPath) + { + string json = File.ReadAllText(jsonPath); + + var options = new JsonSerializerOptions + { + Converters = + { + new JsonStringEnumConverter(JsonNamingPolicy.CamelCase) + }, + PropertyNameCaseInsensitive = true + }; + + var testDefinition = JsonSerializer.Deserialize(json, options); + if (testDefinition == null) + throw new JsonException($"Error while parsing {jsonPath}"); + + testDefinition.Path = Path.GetDirectoryName(jsonPath)!; + return testDefinition; + } + } +} \ No newline at end of file diff --git a/Feature.Detect/FeatureJson/FeatureJson.cs b/Feature.Detect/FeatureJson/FeatureJson.cs new file mode 100644 index 0000000..d62079e --- /dev/null +++ b/Feature.Detect/FeatureJson/FeatureJson.cs @@ -0,0 +1,54 @@ +// /* +// * Copyright 2024 Kelvin Nishikawa +// * +// * Licensed under the Apache License, Version 2.0 (the "License"); +// * you may not use this file except in compliance with the License. +// * You may obtain a copy of the License at +// * +// * http://www.apache.org/licenses/LICENSE-2.0 +// * +// * Unless required by applicable law or agreed to in writing, software +// * distributed under the License is distributed on an "AS IS" BASIS, +// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// * See the License for the specific language governing permissions and +// * limitations under the License. +// */ + +using System.Text.Json.Serialization; + +namespace Feature.Detect.FeatureJson; + +public class FeatureJson +{ + [JsonPropertyName("source")] + public string? Source { get; set; } + + public string? Path { get; set; } + + [JsonPropertyName("module")] + public string? Module { get; set; } + + [JsonPropertyName("options")] + public Options? Options { get; set; } + + [JsonPropertyName("name")] + public string? Name { get; set; } + + [JsonPropertyName("proposal")] + public string? Proposal { get; set; } + + [JsonPropertyName("features")] + public List? Features { get; set; } // New property for features list + + public override string ToString() + { + return $"Name: {Name}\n Proposal: {Proposal}\n Features: {string.Join(", ", Features ?? new List())}\n Module: {Module}\n Options: {Options}"; + } + +} + +public class Options +{ + [JsonPropertyName("builtins")] + public List? Builtins { get; set; } +} \ No newline at end of file diff --git a/Feature.Detect/testsettings.json b/Feature.Detect/testsettings.json new file mode 100644 index 0000000..f8e3efa --- /dev/null +++ b/Feature.Detect/testsettings.json @@ -0,0 +1,3 @@ +{ + "JsonDirectory": "../../../generated-wasm" +} \ No newline at end of file From 35007d26486f6947c43af66d472734eda073edd4 Mon Sep 17 00:00:00 2001 From: Kelvin Nishikawa Date: Sun, 17 Nov 2024 19:50:02 -0800 Subject: [PATCH 5/5] Adding feature detection table publish script --- Feature.Detect/.gitignore | 5 +- Feature.Detect/FeatureJson/FeatureJson.cs | 15 +++- Feature.Detect/build_feature_detect.sh | 6 ++ Feature.Detect/convert_detector.mjs | 5 ++ Feature.Detect/package.json | 4 +- Feature.Detect/trx-to-markdown.js | 84 +++++++++++++++++++++++ README.md | 37 +++++++++- features.md | 24 +++++++ 8 files changed, 175 insertions(+), 5 deletions(-) create mode 100644 Feature.Detect/trx-to-markdown.js create mode 100644 features.md diff --git a/Feature.Detect/.gitignore b/Feature.Detect/.gitignore index 75b7c11..f001fa3 100644 --- a/Feature.Detect/.gitignore +++ b/Feature.Detect/.gitignore @@ -34,4 +34,7 @@ temp/ #generated files -/generated-wasm/** \ No newline at end of file +/generated-wasm/** + +#test run +/TestResults/** \ No newline at end of file diff --git a/Feature.Detect/FeatureJson/FeatureJson.cs b/Feature.Detect/FeatureJson/FeatureJson.cs index d62079e..aa5311a 100644 --- a/Feature.Detect/FeatureJson/FeatureJson.cs +++ b/Feature.Detect/FeatureJson/FeatureJson.cs @@ -14,7 +14,9 @@ // * limitations under the License. // */ +using System.Text; using System.Text.Json.Serialization; +using Microsoft.Extensions.Primitives; namespace Feature.Detect.FeatureJson; @@ -25,6 +27,9 @@ public class FeatureJson public string? Path { get; set; } + [JsonPropertyName("id")] + public string? Id { get; set; } + [JsonPropertyName("module")] public string? Module { get; set; } @@ -42,7 +47,15 @@ public class FeatureJson public override string ToString() { - return $"Name: {Name}\n Proposal: {Proposal}\n Features: {string.Join(", ", Features ?? new List())}\n Module: {Module}\n Options: {Options}"; + var feats = (Features ?? new List()).Select(f => $"\"{f}\""); + var sb = new StringBuilder(); + sb.Append("{") + .Append("\"Name\": \"").Append(Name).Append("\",\n") + .Append("\"Proposal\": \"").Append(Proposal).Append("\",\n") + .Append("\"Features\": [").Append(string.Join(", ", feats)).Append("],\n") + .Append("\"Id\": \"").Append(Id).Append("\"\n") + .Append("}"); + return sb.ToString(); } } diff --git a/Feature.Detect/build_feature_detect.sh b/Feature.Detect/build_feature_detect.sh index 5171f19..ade189a 100755 --- a/Feature.Detect/build_feature_detect.sh +++ b/Feature.Detect/build_feature_detect.sh @@ -16,3 +16,9 @@ for dir in "$PARENT_DIR"/*/; do node convert_detector.mjs "$dir" "$OUTPUT_DIR" fi done + +#run detection +dotnet test --logger "trx;LogFileName=TestResults.trx" + +#publish support matrix +node trx-to-markdown.js TestResults/TestResults.trx ../features.md diff --git a/Feature.Detect/convert_detector.mjs b/Feature.Detect/convert_detector.mjs index 9f25c31..bdb1416 100644 --- a/Feature.Detect/convert_detector.mjs +++ b/Feature.Detect/convert_detector.mjs @@ -90,6 +90,7 @@ async function processWatFile(watPath, outputBasePath) { const metadata = { source: 'wat2wasm', timestamp: new Date().toISOString(), + id: outputFileName, module: outputFilename, ...proposalInfo, }; @@ -150,6 +151,7 @@ const handler = { // console.log(`Checking presence of property "${String(prop)}" in WebAssembly`); writeMetadata({ source: `"${String(prop)}" in WebAssembly`, + id: outputFileName, }, indexPath); // Return the result of the default behavior @@ -165,6 +167,7 @@ webAssemblyProxy.Module = function(bufferSource) { const outfile = writeWasmBuffer(bufferSource, 'Module', indexPath); writeMetadata({ source: 'WebAssembly.Module', + id: outputFileName, module: outfile, }, indexPath); @@ -177,6 +180,7 @@ webAssemblyProxy.instantiate = function(bufferSource, importObject, options) { const metadata = { source: 'WebAssembly.instantiate', + id: outputFileName, module: outfile, options: options || {}, }; @@ -191,6 +195,7 @@ webAssemblyProxy.validate = function(bufferSource) { const metadata = { source: 'WebAssembly.validate', + id: outputFileName, module: outfile, }; diff --git a/Feature.Detect/package.json b/Feature.Detect/package.json index 29db21c..c113cbe 100644 --- a/Feature.Detect/package.json +++ b/Feature.Detect/package.json @@ -1,5 +1,7 @@ { "dependencies": { - "wabt": "^1.0.36" + "he": "^1.2.0", + "wabt": "^1.0.36", + "xml2js": "^0.6.2" } } diff --git a/Feature.Detect/trx-to-markdown.js b/Feature.Detect/trx-to-markdown.js new file mode 100644 index 0000000..ed46326 --- /dev/null +++ b/Feature.Detect/trx-to-markdown.js @@ -0,0 +1,84 @@ +const fs = require('fs').promises; +const xml2js = require('xml2js'); +const path = require('path'); +const he = require("he"); + +class TrxToMarkdown { + constructor() { + this.parser = new xml2js.Parser(); + } + + async convertFile(inputPath, outputPath) { + try { + const xmlData = await fs.readFile(inputPath, 'utf8'); + const result = await this.parser.parseStringPromise(xmlData); + const markdown = this.generateMarkdown(result); + await fs.writeFile(outputPath, markdown); + console.log(`Successfully converted ${inputPath} to ${outputPath}`); + } catch (error) { + console.error('Error converting file:', error); + throw error; + } + } + + generateMarkdown(trxData) { + const testRun = trxData.TestRun; + const results = testRun.Results[0].UnitTestResult; + + let tests = []; + results.forEach(result => { + const test = result['$']; + const jsonStartIndex = test.testName.indexOf('{'); + const jsonString = test.testName.slice(jsonStartIndex, -1); + + function decodeHtmlEntities(str) { + const he = require('he'); + return he.decode(str); + } + + const json = decodeHtmlEntities(jsonString); + const testDef = JSON.parse(json); + testDef['outcome'] = test.outcome; + + tests.push(testDef); + }); + + let markdown = []; + + markdown.push(`|Proposal |Features| |`); + markdown.push(`|------|-------|----|`); + + tests.sort((a, b) => (a.Id || '').localeCompare(b.Id || '')); + + tests.forEach(testDef => { + markdown.push(`|[${testDef['Name']}](${testDef['Proposal']})|${testDef['Features']}|${testDef['outcome'] === 'Failed'?'❌':'✅'}|`); + }); + + return markdown.join('\n'); + } +} + +// CLI implementation +async function main() { + if (process.argv.length !== 4) { + console.log('Usage: node trx-to-markdown.js '); + process.exit(1); + } + + const inputFile = process.argv[2]; + const outputFile = process.argv[3]; + + try { + const converter = new TrxToMarkdown(); + await converter.convertFile(inputFile, outputFile); + } catch (error) { + console.error('Conversion failed:', error); + process.exit(1); + } +} + +if (require.main === module) { + main(); +} + +module.exports = TrxToMarkdown; \ No newline at end of file diff --git a/README.md b/README.md index ce9526a..479a677 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ The chapters and sections from the spec are commented throughout the source code - [Interop Bindings](#interop-bindings) - [Customization](#customization) - [Roadmap](#roadmap) +- [WebAssembly Feature Extensions](#webassembly-feature-extensions) - [License](#license) ## Features @@ -180,7 +181,6 @@ Custom Instruction implementations can be patched in by replacing or inheriting The current TODO list includes: -- **Unity Package Install**: A Unity Asset Store install option. - **ExecAsync**: Thread scheduling and advanced gas metering. - **Wasm Garbage Collection**: Support wasm-gc and heaptypes. - **Text Format Parsing**: Add support for WebAssembly text format. @@ -190,7 +190,40 @@ The current TODO list includes: - **Unity Bindings for SDL**: Implement SDL2 with Unity bindings. - **Instantiation-time Optimization**: Improvements like superinstruction threading and selective inlining for better performance. - **JavaScript Proxy Bindings**: Maybe support common JS env functions. -- **Phase 5 WASM Extensions**: Upcoming features as WebAssembly evolves. + +## WebAssembly Feature Extensions +I started building WACS based on the WebAssembly Core 2 spec, so some of these are already supported. +I'll be implementing and adding support for as many phase 5 features as I can. Depends mostly on complexity and non-javascriptiness. +Here's what's supported so far. + +Harnessed results from [wasm-feature-detect](https://github.com/GoogleChromeLabs/wasm-feature-detect) as compares to [other runtimes](https://webassembly.org/features/): + +|Proposal |Features| | +|------|-------|----| +|[BigInt integration](https://github.com/WebAssembly/JS-BigInt-integration)||✅| +|[Bulk memory operations](https://github.com/webassembly/bulk-memory-operations)||✅| +|[Legacy Exception Handling](https://github.com/WebAssembly/exception-handling)|exceptions|❌| +|[Exception Handling with exnref](https://github.com/WebAssembly/exception-handling)|exceptions|❌| +|[Extented Const Expressesions](https://github.com/WebAssembly/extended-const)|extended_const|❌| +|[Garbage Collection](https://github.com/WebAssembly/gc)|gc|❌| +|[JS String Builtins Proposal for WebAssembly](https://github.com/WebAssembly/js-string-builtins)||❌| +|[JavaScript Promise Integration](https://github.com/WebAssembly/js-promise-integration)|jspi|❌| +|[Memory64](https://github.com/WebAssembly/memory64)|memory64|❌| +|[Multiple Memories](https://github.com/WebAssembly/multi-memory)|multi-memory|❌| +|[Multi-value](https://github.com/WebAssembly/multi-value)|multi_value|✅| +|[Importable/Exportable mutable globals]()||✅| +|[Reference Types](https://github.com/WebAssembly/reference-types)||✅| +|[Relaxed SIMD](https://github.com/webassembly/relaxed-simd)|relaxed_simd|❌| +|[Non-trapping float-to-int conversions](https://github.com/WebAssembly/nontrapping-float-to-int-conversions)||✅| +|[Sign-extension operators](https://github.com/WebAssembly/sign-extension-ops)||✅| +|[Fixed-Width SIMD](https://github.com/webassembly/simd)||✅| +|[Streaming Compilation](https://webassembly.github.io/spec/web-api/index.html#streaming-modules)|streaming_compilation|❌| +|[Tail call](https://github.com/webassembly/tail-call)|tail_call|❌| +|[Threads](https://github.com/webassembly/threads)|threads|❌| +|[Type Reflection](https://github.com/WebAssembly/js-types)|type-reflection|❌| +|[Typed function references](https://github.com/WebAssembly/function-references)|function-references|❌| + +This table was generated with the Feature.Detect test harness. ## Sponsorship & Collaboration diff --git a/features.md b/features.md new file mode 100644 index 0000000..99fd8e6 --- /dev/null +++ b/features.md @@ -0,0 +1,24 @@ +|Proposal |Features| | +|------|-------|----| +|[BigInt integration](https://github.com/WebAssembly/JS-BigInt-integration)||✅| +|[Bulk memory operations](https://github.com/webassembly/bulk-memory-operations)||✅| +|[Legacy Exception Handling](https://github.com/WebAssembly/exception-handling)|exceptions|❌| +|[Exception Handling with exnref](https://github.com/WebAssembly/exception-handling)|exceptions|❌| +|[Extented Const Expressesions](https://github.com/WebAssembly/extended-const)|extended_const|❌| +|[Garbage Collection](https://github.com/WebAssembly/gc)|gc|❌| +|[JS String Builtins Proposal for WebAssembly](https://github.com/WebAssembly/js-string-builtins)||❌| +|[JavaScript Promise Integration](https://github.com/WebAssembly/js-promise-integration)|jspi|❌| +|[Memory64](https://github.com/WebAssembly/memory64)|memory64|❌| +|[Multiple Memories](https://github.com/WebAssembly/multi-memory)|multi-memory|❌| +|[Multi-value](https://github.com/WebAssembly/multi-value)|multi_value|✅| +|[Importable/Exportable mutable globals]()||✅| +|[Reference Types](https://github.com/WebAssembly/reference-types)||✅| +|[Relaxed SIMD](https://github.com/webassembly/relaxed-simd)|relaxed_simd|❌| +|[Non-trapping float-to-int conversions](https://github.com/WebAssembly/nontrapping-float-to-int-conversions)||✅| +|[Sign-extension operators](https://github.com/WebAssembly/sign-extension-ops)||✅| +|[Fixed-Width SIMD](https://github.com/webassembly/simd)||✅| +|[Streaming Compilation](https://webassembly.github.io/spec/web-api/index.html#streaming-modules)|streaming_compilation|❌| +|[Tail call](https://github.com/webassembly/tail-call)|tail_call|❌| +|[Threads](https://github.com/webassembly/threads)|threads|❌| +|[Type Reflection](https://github.com/WebAssembly/js-types)|type-reflection|❌| +|[Typed function references](https://github.com/WebAssembly/function-references)|function-references|❌| \ No newline at end of file