Introduce parser for dynamic token system

lib/load.php

@@ -151,6 +151,7 @@ function gutenberg_is_experiment_enabled( $name ) {
 require __DIR__ . '/client-assets.php';
 require __DIR__ . '/demo.php';
 require __DIR__ . '/experiments-page.php';
+require __DIR__ . '/tokens.php';
 
 // Copied package PHP files.
 if ( file_exists( __DIR__ . '/../build/style-engine/class-wp-style-engine-gutenberg.php' ) ) {

lib/tokens.php

@@ -0,0 +1,67 @@
+<?php
+/**
+ * Token functions specific for the Gutenberg editor plugin.
+ *
+ * @package gutenberg
+ */
+
+
+function register_token( $namespace, $name, $replacer, $priority = 10 ) {
+	add_filter(
+		'render_token',
+		function ( $rendered, $token ) use ( $namespace, $name, $replacer ) {
+			if ( ! ( $namespace === $token->namespace && $name === $token->name ) ) {
+				return $rendered;
+			}
+
+			return call_user_func( $replacer, $rendered, $token );
+		},
+		$priority,
+		2
+	);
+}
+
+
+function tokens_core_replacer( $rendered, $token ) {
+	global $post_id;
+
+	if ( 'core' !== $token->namespace ) {
+		return $rendered;
+	}
+
+	switch ( $token->name ) {
+		case 'featured-image':
+			return get_the_post_thumbnail_url();
+
+		case 'permalink':
+			$permalink = get_permalink( is_int( $token->value ) ? $token->value : $post_id );
+
+			return false !== $permalink ? $permalink : '';
+
+		case 'plugins_url':
+			return plugins_url();
+
+		default:
+			return $token->fallback;
+	}
+}
+
+
+function tokens_token_replacer( $token ) {
+	return apply_filters( 'render_token', $token->fallback, $token );
+}
+
+
+function tokens_swap_tokens( $text ) {
+	return WP_Token_parser::swap_tokens( 'tokens_token_replacer', $text );
+}
+
+
+function tokens_init() {
+	require_once __DIR__ . '/../packages/tokens/token-parser.php';
+
+	add_filter( 'render_token', 'tokens_core_replacer', 2, 10 );
+	add_filter( 'the_content', 'tokens_swap_tokens', 1, 1000 );
+}
+
+add_action( 'init', 'tokens_init' );

packages/tokens/README.md

@@ -0,0 +1,84 @@
+# Tokens
+
+This module provides the functionality necessary to implement a dynamic-token replacement system.
+Dynamic tokens are simple and non-nested markers within a document that refer to externally-sourced data.
+On page render they are to be replaced with some server-side function.
+During edit sessions they should appear visually as tokens and not as their replaced values.
+Tokens should be replaced with basic textual content.
+
+## Usage
+
+Because tokens are designed to support human entry their syntax is more flexible than block comment delimiters.
+There are multiple ways to enter tokens that have different levels of convenience for the varying amounts of their required information.
+
+### Token properties
+
+| Name | Data Type | Description |
+|---|---|---|
+| `token` | Identifier | Indicates the `name` of a token as its `namespace`, or `core` if the namespace is omitted. e.g. `featured-image` or `my-plugin/weather`. |
+| `value` | `any` | Data passed to token rendering function as an argument. Some tokens might need more than just a name to properly render. e.g. `{"stat": "temperature", "unit": "C"}` |
+| `fallback` | `string` | Plaintext string to render in place of the token if no suitable renderer is avialable. This may occur after uninstalling a plugin or when importing content from another site. Without a fallback tokens will be removed from the document to prevent leaking internal or private information. |
+
+### Token sytnax
+
+Choosing an appropriate token syntax is mostly an exercise in determining how much information is necessary to render the token.
+The more data we need, the more sytnax we also need.
+Likewise the less data we need to convey, the terser we can write the token.
+
+#### Namespacing and the `core` namespace
+
+Tokens are identified by their `namespace` and `name` pair.
+For example, `my-plugin/weather` identifies the `weather` token in the `my-plugin` namespace.
+Typically the `namespace` corresponds to the WordPress plugin which registered the token.
+The `core` namespace is special and represents the functionality built in to WordPress.
+It's possible to omit the `core/` namespace prefix when identifying a token.
+Likewise, if a token lacks a namespace it's implied to be provided in the `core` namespace.
+
+#### Simple tokens
+
+Tokens that don't require any arguments can be entered with their unescaped names.
+
+| Syntax | Meaning                                   |
+|---|-------------------------------------------|
+| `#{featured-image}#` | Display the `core/featured-image` token |
+| `#{core/featured-image}#` | Display the `core/featured-iamge}` token |
+| `#{retro/page-counter}#` | Display the `retro/page-counter` token |
+
+
+#### Simple tokens with arguments
+
+If your token needs additional information in order to properly render you can pass them in as augmented JSON.
+
+| Syntax | Meaning                                   |
+|---|-------------------------------------------|
+| `#{permalink=14}#` | Display the permalink for the post whose id is 14 |
+| `#{my-plugin/weather={"stat": "temperature", "units": "C"}}#` | Display the temperature in ºC |
+| `#{echo="\u{3c}span\u{3e}\u{23}1\u{3c}/span\u{3e}"}#` | Render `<span>#1</span>` |
+
+#### Tokens with fallback
+
+If a token provides a fallback value it must use the fully-verbose syntax.
+It's always possible to use the full syntax.
+
+| Syntax | Meaning                                   |
+|---|-------------------------------------------|
+| `#{"token":"sportier/stat", "value":{"sport": "basketball", "game": "latest"}, "fallback": "TBD"}}#` | Show the score for the latest basketball game, but the plugin isn't available render `TBD` instead. |
+
+#### Indicating context for token
+
+While not currently supported, it may arise that we need to indicate in which context the token is found.
+This is due to the fact that there are different escaping and security concerns for content bound for HTML attributes than there are for those bound for HTML markup, similarly for other potential contexts.
+
+Tokens support indicating this with a _sigil_ at the front of the token syntax according to the following table.
+Not all potential sigils are meaningful, and in the absence of a recognized sigil the context for a token remains blank  (`null`).
+
+| Sigil | Associated context                                     |
+|---|--------------------------------------------------------|
+| `a` | token is inside an HTML attribute                      |
+| `h` | token is inside normal HTML markup                     |
+| `j` | token is inside a `<script>` tag or in JavaScript code |
+
+The context is a hint to the backend on how to render and escape the token content.
+It's not yet decided if these will be enforced by the token system or left up to the token authors to enforce.
+
+## Background

packages/tokens/test/index.js

@@ -0,0 +1,28 @@
+/**
+ * External dependencies
+ */
+import path from 'path';
+
+/**
+ * Internal dependencies
+ */
+import { swapTokens } from '../token-parser';
+import { jsTester, phpTester } from './shared.test';
+
+const parse = ( input ) => {
+	let count = 0;
+	const tokens = [];
+
+	const tokenReplacer = ( token ) => {
+		tokens.push( token );
+		return `{{TOKEN_${ ++count }}}`;
+	};
+
+	const output = swapTokens( tokenReplacer, input );
+
+	return { tokens, output };
+};
+
+describe( 'tokens', jsTester( parse ) ); // eslint-disable-line jest/valid-describe-callback
+
+phpTester( 'token-parser-php', path.join( __dirname, 'test-parser.php' ) );

packages/tokens/test/shared.test.js

@@ -0,0 +1,180 @@
+// eslint-disable-next-line jest/no-export
+export const jsTester = ( parse ) => () => {
+	const tokenForms = [
+		[
+			'#{echo}#',
+			{
+				namespace: 'core',
+				name: 'echo',
+				value: null,
+				fallback: '',
+				context: null,
+			},
+		],
+		[
+			'#{core/identity}#',
+			{
+				namespace: 'core',
+				name: 'identity',
+				value: null,
+				fallback: '',
+				context: null,
+			},
+		],
+		[
+			'#{"token":"core/identity"}#',
+			{
+				namespace: 'core',
+				name: 'identity',
+				value: null,
+				fallback: '',
+				context: null,
+			},
+		],
+		[
+			'#{echo="\u{3c}"}#',
+			{
+				namespace: 'core',
+				name: 'echo',
+				value: '<',
+				fallback: '',
+				context: null,
+			},
+		],
+		[
+			'#{"token":"my_plugin/widget", "value": {"name": "sprocket"}, "fallback": "just a sprocket"}#',
+			{
+				namespace: 'my_plugin',
+				name: 'widget',
+				value: { name: 'sprocket' },
+				fallback: 'just a sprocket',
+				context: null,
+			},
+		],
+		[
+			'#{my_plugin/widget={"name":"sprocket"}}#',
+			{
+				namespace: 'my_plugin',
+				name: 'widget',
+				value: { name: 'sprocket' },
+				fallback: '',
+				context: null,
+			},
+		],
+	];
+
+	it.each( tokenForms )( 'basic token syntax: %s', ( input, token ) => {
+		const output = parse( input );
+		expect( output ).toHaveProperty( 'tokens', [ token ] );
+		expect( output ).toHaveProperty( 'output', '{{TOKEN_1}}' );
+	} );
+
+	it( 'avoids escaping quoted tokens', () => {
+		expect( parse( '##{not-a-token}#' ) ).toEqual( {
+			tokens: [],
+			output: '#{not-a-token}#',
+		} );
+	} );
+
+	it( 'fails to parse when # is unescaped inside of token', () => {
+		expect( parse( '#{echo="look out for #1"}#' ) ).toEqual( {
+			tokens: [],
+			output: '#{echo="look out for #1"}#',
+		} );
+
+		expect( parse( '#{echo="look out for \\u00231"}#' ) ).toEqual( {
+			tokens: [
+				{
+					namespace: 'core',
+					name: 'echo',
+					value: 'look out for #1',
+					fallback: '',
+					context: null,
+				},
+			],
+			output: '{{TOKEN_1}}',
+		} );
+	} );
+
+	it.each( [
+		[ '#a{echo}#', 'attribute' ],
+		[ '#h{echo}#', 'html' ],
+		[ '#j{echo}#', 'javascript' ],
+		[ '#z{echo}#', null ],
+		[ '#3{echo}#', null ],
+		[ '#{echo}#', null ],
+	] )( 'recognizes context-specifying sigils: %s', ( input, context ) => {
+		expect( parse( input ).tokens[ 0 ] ).toHaveProperty(
+			'context',
+			context
+		);
+	} );
+};
+
+const hasPHP =
+	'test' === process.env.NODE_ENV
+		? ( () => {
+				const process = require( 'child_process' ).spawnSync(
+					'php',
+					[ '-r', 'echo 1;' ],
+					{
+						encoding: 'utf8',
+					}
+				);
+
+				return process.status === 0 && process.stdout === '1';
+		  } )()
+		: false;
+
+// Skipping if `php` isn't available to us, such as in local dev without it
+// skipping preserves snapshots while commenting out or simply
+// not injecting the tests prompts `jest` to remove "obsolete snapshots"
+const makeTest = hasPHP
+	? // eslint-disable-next-line jest/valid-describe-callback, jest/valid-title
+	  ( ...args ) => describe( ...args )
+	: // eslint-disable-next-line jest/no-disabled-tests, jest/valid-describe-callback, jest/valid-title
+	  ( ...args ) => describe.skip( ...args );
+
+// eslint-disable-next-line jest/no-export
+export const phpTester = ( name, filename ) =>
+	makeTest(
+		name,
+		'test' === process.env.NODE_ENV
+			? jsTester( ( doc ) => {
+					const process = require( 'child_process' ).spawnSync(
+						'php',
+						[ '-f', filename ],
+						{
+							input: doc,
+							encoding: 'utf8',
+							timeout: 30 * 1000, // Abort after 30 seconds, that's too long anyway.
+						}
+					);
+
+					if ( process.status !== 0 ) {
+						throw new Error( process.stderr || process.stdout );
+					}
+
+					try {
+						/*
+						 * Due to an issue with PHP's json_encode() serializing an empty associative array
+						 * as an empty list `[]` we're manually replacing the already-encoded bit here.
+						 *
+						 * This is an issue with the test runner, not with the parser.
+						 */
+						return JSON.parse(
+							process.stdout.replace(
+								/"attributes":\s*\[\]/g,
+								'"attributes":{}'
+							)
+						);
+					} catch ( e ) {
+						// eslint-disable-next-line no-console
+						console.error( process.stdout );
+						throw new Error(
+							'failed to parse JSON:\n' + process.stdout
+						);
+					}
+			  } )
+			: () => {}
+	);

packages/tokens/test/test-parser.php

@@ -0,0 +1,24 @@
+<?php
+/**
+ * PHP Test Helper
+ *
+ * Facilitates running PHP parser against same tests as the JS parser implementation.
+ *
+ * @package gutenberg
+ */
+
+// Include the generated parser.
+require_once __DIR__ . '/../token-parser.php';
+
+$count = 0;
+$tokens = [];
+
+$output = WP_Token_Parser::swap_tokens(
+	function ( $token ) use ( &$count, &$tokens ) {
+		$tokens[] = $token;
+		return '{{TOKEN_' . ++$count . '}}';
+	},
+	file_get_contents( 'php://stdin' )
+);
+
+echo json_encode( [ 'tokens' => $tokens, 'output' => $output ] );