diff --git a/CHANGELOG.md b/CHANGELOG.md
index 33b20469135..34c3ff0a6f4 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,30 +4,29 @@ All notable changes to this project are documented in this file by a CI job
that runs on every NPM release. The file follows the [Keep a Changelog](https://keepachangelog.com/en/1.1.0/)
format.
-## [v1.0.14] (2024-12-02)
+## [v1.0.14] (2024-12-02)
### Blueprints
-- Resolve the latest WordPress version from the API instead of assuming it's the same as the last minified build. ([#2027](https://github.com/WordPress/wordpress-playground/pull/2027))
+- Resolve the latest WordPress version from the API instead of assuming it's the same as the last minified build. ([#2027](https://github.com/WordPress/wordpress-playground/pull/2027))
### Tools
-
#### Blueprints Builder
-- Add installPlugin support for single plugin files. ([#2033](https://github.com/WordPress/wordpress-playground/pull/2033))
+- Add installPlugin support for single plugin files. ([#2033](https://github.com/WordPress/wordpress-playground/pull/2033))
### PHP WebAssembly
-- Networking: Preserve the content-type header when fetch()-ing. ([#2028](https://github.com/WordPress/wordpress-playground/pull/2028))
+- Networking: Preserve the content-type header when fetch()-ing. ([#2028](https://github.com/WordPress/wordpress-playground/pull/2028))
### Website
-- [Web] Re-enable wp-cron. ([#2039](https://github.com/WordPress/wordpress-playground/pull/2039))
+- [Web] Re-enable wp-cron. ([#2039](https://github.com/WordPress/wordpress-playground/pull/2039))
### Various
-- [Data Liberation] WP_Stream_Importer: User-driven incremental import. ([#2013](https://github.com/WordPress/wordpress-playground/pull/2013))
+- [Data Liberation] WP_Stream_Importer: User-driven incremental import. ([#2013](https://github.com/WordPress/wordpress-playground/pull/2013))
### Contributors
@@ -35,7 +34,6 @@ The following contributors merged PRs in this release:
@adamziel @brandonpayton
-
## [v1.0.13] (2024-11-25)
### Enhancements
diff --git a/packages/docs/site/docs/main/changelog.md b/packages/docs/site/docs/main/changelog.md
index aef43b75000..53f538b8aa3 100644
--- a/packages/docs/site/docs/main/changelog.md
+++ b/packages/docs/site/docs/main/changelog.md
@@ -9,30 +9,29 @@ All notable changes to this project are documented in this file by a CI job
that runs on every NPM release. The file follows the [Keep a Changelog](https://keepachangelog.com/en/1.1.0/)
format.
-## [v1.0.14] (2024-12-02)
+## [v1.0.14] (2024-12-02)
### Blueprints
-- Resolve the latest WordPress version from the API instead of assuming it's the same as the last minified build. ([#2027](https://github.com/WordPress/wordpress-playground/pull/2027))
+- Resolve the latest WordPress version from the API instead of assuming it's the same as the last minified build. ([#2027](https://github.com/WordPress/wordpress-playground/pull/2027))
### Tools
-
#### Blueprints Builder
-- Add installPlugin support for single plugin files. ([#2033](https://github.com/WordPress/wordpress-playground/pull/2033))
+- Add installPlugin support for single plugin files. ([#2033](https://github.com/WordPress/wordpress-playground/pull/2033))
### PHP WebAssembly
-- Networking: Preserve the content-type header when fetch()-ing. ([#2028](https://github.com/WordPress/wordpress-playground/pull/2028))
+- Networking: Preserve the content-type header when fetch()-ing. ([#2028](https://github.com/WordPress/wordpress-playground/pull/2028))
### Website
-- [Web] Re-enable wp-cron. ([#2039](https://github.com/WordPress/wordpress-playground/pull/2039))
+- [Web] Re-enable wp-cron. ([#2039](https://github.com/WordPress/wordpress-playground/pull/2039))
### Various
-- [Data Liberation] WP_Stream_Importer: User-driven incremental import. ([#2013](https://github.com/WordPress/wordpress-playground/pull/2013))
+- [Data Liberation] WP_Stream_Importer: User-driven incremental import. ([#2013](https://github.com/WordPress/wordpress-playground/pull/2013))
### Contributors
@@ -40,7 +39,6 @@ The following contributors merged PRs in this release:
@adamziel @brandonpayton
-
## [v1.0.13] (2024-11-25)
### Enhancements
diff --git a/packages/php-wasm/compile/php/Dockerfile b/packages/php-wasm/compile/php/Dockerfile
index 1c4c9f94dcf..0c5bef72150 100644
--- a/packages/php-wasm/compile/php/Dockerfile
+++ b/packages/php-wasm/compile/php/Dockerfile
@@ -569,6 +569,7 @@ export ASYNCIFY_ONLY=$'"rc_dtor_func",\
"_php_stream_fill_read_buffer",\
"_php_stream_read",\
"php_stream_read_to_str",\
+"php_userstreamop_read",\
"zif_fread",\
"wasm_read",\
"php_stdiop_read",\
diff --git a/packages/php-wasm/node/asyncify/8_3_0/php_8_3.wasm b/packages/php-wasm/node/asyncify/8_3_0/php_8_3.wasm
index 8b8b0c94d69..5c35cb2d32f 100755
Binary files a/packages/php-wasm/node/asyncify/8_3_0/php_8_3.wasm and b/packages/php-wasm/node/asyncify/8_3_0/php_8_3.wasm differ
diff --git a/packages/php-wasm/node/asyncify/php_8_3.js b/packages/php-wasm/node/asyncify/php_8_3.js
index fdd708b4594..8d6e96343e1 100644
--- a/packages/php-wasm/node/asyncify/php_8_3.js
+++ b/packages/php-wasm/node/asyncify/php_8_3.js
@@ -1,6 +1,6 @@
const dependencyFilename = __dirname + '/8_3_0/php_8_3.wasm';
export { dependencyFilename };
-export const dependenciesTotalSize = 15083501;
+export const dependenciesTotalSize = 15085661;
export function init(RuntimeName, PHPLoader) {
/**
* Overrides Emscripten's default ExitStatus object which gets
diff --git a/packages/playground/data-liberation/bootstrap.php b/packages/playground/data-liberation/bootstrap.php
index 2bd3420eaa1..8e4e5f0177e 100644
--- a/packages/playground/data-liberation/bootstrap.php
+++ b/packages/playground/data-liberation/bootstrap.php
@@ -17,25 +17,25 @@
require_once __DIR__ . '/src/byte-readers/WP_Remote_File_Reader.php';
require_once __DIR__ . '/src/byte-readers/WP_Remote_File_Ranged_Reader.php';
-if(!class_exists('WP_HTML_Tag_Processor')) {
- require_once __DIR__ . "/src/wordpress-core-html-api/class-wp-html-token.php";
- require_once __DIR__ . "/src/wordpress-core-html-api/class-wp-html-span.php";
- require_once __DIR__ . "/src/wordpress-core-html-api/class-wp-html-text-replacement.php";
- require_once __DIR__ . "/src/wordpress-core-html-api/class-wp-html-decoder.php";
- require_once __DIR__ . "/src/wordpress-core-html-api/class-wp-html-attribute-token.php";
-
- require_once __DIR__ . "/src/wordpress-core-html-api/class-wp-html-decoder.php";
- require_once __DIR__ . "/src/wordpress-core-html-api/class-wp-html-tag-processor.php";
- require_once __DIR__ . "/src/wordpress-core-html-api/class-wp-html-open-elements.php";
- require_once __DIR__ . "/src/wordpress-core-html-api/class-wp-token-map.php";
- require_once __DIR__ . "/src/wordpress-core-html-api/html5-named-character-references.php";
- require_once __DIR__ . "/src/wordpress-core-html-api/class-wp-html-active-formatting-elements.php";
- require_once __DIR__ . "/src/wordpress-core-html-api/class-wp-html-processor-state.php";
- require_once __DIR__ . "/src/wordpress-core-html-api/class-wp-html-unsupported-exception.php";
- require_once __DIR__ . "/src/wordpress-core-html-api/class-wp-html-processor.php";
+if ( ! class_exists( 'WP_HTML_Tag_Processor' ) ) {
+ require_once __DIR__ . '/src/wordpress-core-html-api/class-wp-html-token.php';
+ require_once __DIR__ . '/src/wordpress-core-html-api/class-wp-html-span.php';
+ require_once __DIR__ . '/src/wordpress-core-html-api/class-wp-html-text-replacement.php';
+ require_once __DIR__ . '/src/wordpress-core-html-api/class-wp-html-decoder.php';
+ require_once __DIR__ . '/src/wordpress-core-html-api/class-wp-html-attribute-token.php';
+
+ require_once __DIR__ . '/src/wordpress-core-html-api/class-wp-html-decoder.php';
+ require_once __DIR__ . '/src/wordpress-core-html-api/class-wp-html-tag-processor.php';
+ require_once __DIR__ . '/src/wordpress-core-html-api/class-wp-html-open-elements.php';
+ require_once __DIR__ . '/src/wordpress-core-html-api/class-wp-token-map.php';
+ require_once __DIR__ . '/src/wordpress-core-html-api/html5-named-character-references.php';
+ require_once __DIR__ . '/src/wordpress-core-html-api/class-wp-html-active-formatting-elements.php';
+ require_once __DIR__ . '/src/wordpress-core-html-api/class-wp-html-processor-state.php';
+ require_once __DIR__ . '/src/wordpress-core-html-api/class-wp-html-unsupported-exception.php';
+ require_once __DIR__ . '/src/wordpress-core-html-api/class-wp-html-processor.php';
}
-if (!isset($html5_named_character_references)) {
- require_once __DIR__ . "/src/wordpress-core-html-api/html5-named-character-references.php";
+if ( ! isset( $html5_named_character_references ) ) {
+ require_once __DIR__ . '/src/wordpress-core-html-api/html5-named-character-references.php';
}
require_once __DIR__ . '/src/block-markup/WP_Block_Markup_Processor.php';
@@ -56,6 +56,8 @@
require_once __DIR__ . '/src/import/WP_Import_Session.php';
require_once __DIR__ . '/src/import/WP_Stream_Importer.php';
require_once __DIR__ . '/src/import/WP_Markdown_Importer.php';
+require_once __DIR__ . '/src/import/WP_Entity_Iterator_Chain.php';
+require_once __DIR__ . '/src/import/WP_Retry_Frontloading_Iterator.php';
require_once __DIR__ . '/src/utf8_decoder.php';
@@ -65,50 +67,50 @@
require_once __DIR__ . '/vendor/autoload.php';
// Polyfill WordPress core functions
-if (!function_exists('_doing_it_wrong')) {
- $GLOBALS['_doing_it_wrong_messages'] = [];
- function _doing_it_wrong($method, $message, $version) {
+if ( ! function_exists( '_doing_it_wrong' ) ) {
+ $GLOBALS['_doing_it_wrong_messages'] = array();
+ function _doing_it_wrong( $method, $message, $version ) {
$GLOBALS['_doing_it_wrong_messages'][] = $message;
}
}
-if(!function_exists('wp_kses_uri_attributes')) {
+if ( ! function_exists( 'wp_kses_uri_attributes' ) ) {
function wp_kses_uri_attributes() {
- return [];
+ return array();
}
}
-if (!function_exists('__')) {
- function __($input) {
+if ( ! function_exists( '__' ) ) {
+ function __( $input ) {
return $input;
}
}
-if (!function_exists('esc_attr')) {
- function esc_attr($input) {
- return htmlspecialchars($input);
+if ( ! function_exists( 'esc_attr' ) ) {
+ function esc_attr( $input ) {
+ return htmlspecialchars( $input );
}
}
-if (!function_exists('esc_html')) {
- function esc_html($input) {
- return htmlspecialchars($input);
+if ( ! function_exists( 'esc_html' ) ) {
+ function esc_html( $input ) {
+ return htmlspecialchars( $input );
}
}
-if (!function_exists('esc_url')) {
- function esc_url($url) {
- return htmlspecialchars($url);
+if ( ! function_exists( 'esc_url' ) ) {
+ function esc_url( $url ) {
+ return htmlspecialchars( $url );
}
}
-if (!function_exists('wp_kses_uri_attributes')) {
+if ( ! function_exists( 'wp_kses_uri_attributes' ) ) {
function wp_kses_uri_attributes() {
return array();
}
}
-if (!function_exists('mbstring_binary_safe_encoding')) {
+if ( ! function_exists( 'mbstring_binary_safe_encoding' ) ) {
function mbstring_binary_safe_encoding( $reset = false ) {
static $encodings = array();
static $overloaded = null;
@@ -140,7 +142,7 @@ function mbstring_binary_safe_encoding( $reset = false ) {
}
}
-if (!function_exists('reset_mbstring_encoding')) {
+if ( ! function_exists( 'reset_mbstring_encoding' ) ) {
function reset_mbstring_encoding() {
mbstring_binary_safe_encoding( true );
}
diff --git a/packages/playground/data-liberation/data-liberation-page.php b/packages/playground/data-liberation/data-liberation-page.php
new file mode 100644
index 00000000000..d91da6f39db
--- /dev/null
+++ b/packages/playground/data-liberation/data-liberation-page.php
@@ -0,0 +1,331 @@
+
+
+
Data Liberation
+
+
+
Current Import
+
:
+
+
+
Stage:
+
+
+
+
+
+
+
Indexing Entities
+
+
+
+
+ | Entity |
+ Imported |
+ Total |
+
+
+
+
+
+
+
+
+
+
Downloading Assets
+
+
+
+
/
+
Files Downloaded
+
+
Downloads in progress:
+
+
+
+ | File |
+ Progress |
+
+
+
+
+
+ |
+
+
+ /
+
+ |
+
+
+
+
+
+
+
+
+
+
+ | ID |
+ File |
+ Retries |
+ Status |
+
+
+
+
+
+ |
+ |
+ |
+
+
+
+
+
+
+
+
+
+
+
+
+
+ |
+
+
+
+
+
+
+
+
+
Importing Entities
+
+
+
+
+ | Entity |
+ Imported |
+ Total |
+
+
+
+
+
+ |
+ |
+ |
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Previous Import Sessions
+
+
+
\ No newline at end of file
diff --git a/packages/playground/data-liberation/import-screen.js b/packages/playground/data-liberation/import-screen.js
index 808a0db6a83..7f10b6ec9ce 100644
--- a/packages/playground/data-liberation/import-screen.js
+++ b/packages/playground/data-liberation/import-screen.js
@@ -1,14 +1,151 @@
-import { store, getContext } from '@wordpress/interactivity';
-const { state } = store('dataLiberation', {
+/**
+ * @TODO
+ * - Replace window.prompt() calls with a pretty UI-based dialog or
+ * an inline form or something.
+ */
+import { store, getContext, getServerContext } from '@wordpress/interactivity';
+
+const apiFetch = window.wp.apiFetch;
+
+const { state, actions } = store('dataLiberation', {
state: {
- selectedImportType: 'wxr_file',
get isImportTypeSelected() {
- return getContext().importType === state.selectedImportType;
+ return getContext()?.importType === state.selectedImportType;
+ },
+ get frontloadingFailed() {
+ return getContext()?.item.post_status === 'error';
+ },
+ get isCurrentImportAtStage() {
+ return getContext()?.stage.id === state.currentImport.stage;
+ },
+ },
+ /**
+ * We're bombarding the server with HTTP requests both to run the import and to
+ * refresh the reported progress. Do we need such an aggressive refresh rate?
+ */
+ callbacks: {
+ /**
+ * Fetches a fresh interactivity state from the server every second.
+ */
+ async startRefreshingProgress() {
+ while (true) {
+ if (
+ !state.currentImport.active ||
+ state.currentImport.stage === '#finished'
+ ) {
+ await new Promise((resolve) => setTimeout(resolve, 1000));
+ continue;
+ }
+
+ const response = await apiFetch({
+ path: '/data-liberation/v1/interactivity-state',
+ });
+ Object.assign(state, response);
+
+ await new Promise((resolve) => setTimeout(resolve, 2000));
+ }
+ },
+ /**
+ * Continuously asks the server to continue the import.
+ *
+ * @TODO: Ensure two parallel requests are never processing the same import.
+ * That would lead to race conditions, undefined states, bad stuff in general.
+ */
+ async startImportLoop() {
+ let failuresInARow = 0;
+ const maxFailures = 3;
+ while (true) {
+ if (
+ !state.currentImport.active ||
+ state.currentImport.stage === '#finished'
+ ) {
+ await new Promise((resolve) => setTimeout(resolve, 1000));
+ continue;
+ }
+
+ try {
+ const response = await fetch(
+ `${window.location.pathname}?page=data-liberation&continue=true`,
+ {
+ credentials: 'same-origin', // Preserves cookies
+ }
+ );
+ const text = await response.text();
+ const parser = new DOMParser();
+ const doc = parser.parseFromString(text, 'text/html');
+ const importOutput = doc.querySelector('#import-output');
+ if (importOutput) {
+ document.querySelector('#import-output').innerHTML =
+ importOutput.innerHTML;
+ }
+ failuresInARow = 0;
+ } catch (error) {
+ failuresInARow++;
+ // TODO: notify the user about the problem.
+ if (failuresInARow >= maxFailures) {
+ throw error;
+ }
+ }
+ await new Promise((resolve) => setTimeout(resolve, 1000));
+ }
},
},
actions: {
setImportType: () => {
- state.selectedImportType = getContext().importType;
+ if (getContext()) {
+ state.selectedImportType = getContext().importType;
+ }
+ },
+
+ async archiveImport() {
+ window.location.href = `${window.location.pathname}?page=data-liberation&archive=true`;
+ },
+
+ // Existing download management actions
+ async retryDownload(event) {
+ const postId = event.target.dataset.postId;
+ const response = await apiFetch({
+ path: '/data-liberation/v1/retry-download',
+ method: 'POST',
+ data: { post_id: postId },
+ });
+
+ if (response.success) {
+ window.location.reload();
+ }
+ },
+
+ async ignoreDownload(event) {
+ const postId = event.target.dataset.postId;
+ const response = await apiFetch({
+ path: '/data-liberation/v1/ignore-download',
+ method: 'POST',
+ data: { post_id: postId },
+ });
+
+ if (response.success) {
+ window.location.reload();
+ }
+ },
+
+ async changeDownloadUrl(event) {
+ const postId = event.target.dataset.postId;
+ const newUrl = prompt('Enter the new URL for this asset:');
+
+ if (!newUrl) return;
+
+ const response = await apiFetch({
+ path: '/data-liberation/v1/change-download-url',
+ method: 'POST',
+ data: {
+ post_id: postId,
+ new_url: newUrl,
+ },
+ });
+
+ if (response.success) {
+ window.location.reload();
+ }
},
},
});
diff --git a/packages/playground/data-liberation/plugin.php b/packages/playground/data-liberation/plugin.php
index 3c16f5f2e21..94c2b1b3ba8 100644
--- a/packages/playground/data-liberation/plugin.php
+++ b/packages/playground/data-liberation/plugin.php
@@ -2,6 +2,14 @@
/**
* Plugin Name: Data Liberation
* Description: Data parsing and importing primitives.
+ *
+ * @TODO
+ * - Get nonces to work
+ * - Visually appealing UI with smooth transitions between state updates. Right
+ * now the UI is jerky. We could have beautiful CSS transitions that would
+ * make the import process feel much smoother.
+ * - Delete frontloading placeholders that have been successfully downloaded.
+ * Still keep track of the number of total and successful downloads.
*/
require_once __DIR__ . '/bootstrap.php';
@@ -24,632 +32,628 @@
* > echo $html->get_updated_html();
*
*/
-add_filter('wp_kses_uri_attributes', function() {
- return [];
-});
-
-add_action('init', function() {
- if ( defined( 'WP_CLI' ) && WP_CLI ) {
- /**
- * Import a WXR file.
- *
- *
- * : The WXR file to import.
- */
- $command = function ( $args, $assoc_args ) {
- $file = $args[0];
- data_liberation_import( $file );
- };
-
- // Register the WP-CLI import command.
- // Example usage: wp data-liberation /path/to/file.xml
- WP_CLI::add_command( 'data-liberation', $command );
- }
-});
+add_filter(
+ 'wp_kses_uri_attributes',
+ function () {
+ return array();
+ }
+);
+
+add_action(
+ 'init',
+ function () {
+ if ( defined( 'WP_CLI' ) && WP_CLI ) {
+ /**
+ * Import a WXR file.
+ *
+ *
+ * : The WXR file to import.
+ */
+ $command = function ( $args, $assoc_args ) {
+ $file = $args[0];
+ data_liberation_import( $file );
+ };
+
+ // Register the WP-CLI import command.
+ // Example usage: wp data-liberation /path/to/file.xml
+ WP_CLI::add_command( 'data-liberation', $command );
+ }
+
+ register_post_status(
+ 'error',
+ array(
+ 'label' => _x( 'Error', 'post' ), // Label name
+ 'public' => false,
+ 'exclude_from_search' => false,
+ 'show_in_admin_all_list' => false,
+ 'show_in_admin_status_list' => false,
+ // translators: %s is the number of errors
+ 'label_count' => _n_noop( 'Error (%s)', 'Error (%s)' ),
+ )
+ );
+ }
+);
// Register admin menu
-add_action('admin_menu', function() {
- add_menu_page(
- 'Data Liberation',
- 'Data Liberation',
- 'manage_options',
- 'data-liberation',
- 'data_liberation_admin_page',
- 'dashicons-database-import'
- );
-});
-
-add_action('admin_enqueue_scripts', 'enqueue_data_liberation_scripts');
+add_action(
+ 'admin_menu',
+ function () {
+ add_menu_page(
+ 'Data Liberation',
+ 'Data Liberation',
+ 'manage_options',
+ 'data-liberation',
+ 'data_liberation_admin_page',
+ 'dashicons-database-import'
+ );
+ }
+);
+
+add_action( 'admin_enqueue_scripts', 'enqueue_data_liberation_scripts' );
function enqueue_data_liberation_scripts() {
- wp_register_script_module(
- '@data-liberation/import-screen',
- plugin_dir_url( __FILE__ ) . 'import-screen.js',
- array( '@wordpress/interactivity' )
- );
- wp_enqueue_script_module(
- '@data-liberation/import-screen',
- plugin_dir_url( __FILE__ ) . 'import-screen.js',
- array( '@wordpress/interactivity' )
- );
+ wp_register_script_module(
+ '@data-liberation/import-screen',
+ plugin_dir_url( __FILE__ ) . 'import-screen.js',
+ array( '@wordpress/interactivity', '@wordpress/interactivity-router', 'wp-api-fetch' )
+ );
+ wp_enqueue_script( 'wp-api-fetch' );
+ wp_enqueue_script_module(
+ '@data-liberation/import-screen',
+ plugin_dir_url( __FILE__ ) . 'import-screen.js',
+ array( '@wordpress/interactivity', '@wordpress/interactivity-router' )
+ );
}
function data_liberation_add_minute_schedule( $schedules ) {
- // add a 'weekly' schedule to the existing set
- $schedules['data_liberation_minute'] = array(
- 'interval' => 60,
- 'display' => __('Once a Minute')
- );
- return $schedules;
+ // add a 'weekly' schedule to the existing set
+ $schedules['data_liberation_minute'] = array(
+ 'interval' => 30,
+ 'display' => __( 'Twice a Minute' ),
+ );
+ return $schedules;
}
add_filter( 'cron_schedules', 'data_liberation_add_minute_schedule' );
// Render admin page
function data_liberation_admin_page() {
- $import_session = WP_Import_Session::get_active();
- if($import_session) {
- if(isset($_GET['archive'])) {
- $import_session->archive();
- echo '';
- exit;
- } elseif(isset($_GET['continue'])) {
- echo 'Next import step stdout output:
';
- echo '';
- data_liberation_process_import();
- echo '
';
- }
- }
-
- // Populates the initial global state values.
- wp_interactivity_state( 'dataLiberation', array(
- 'selectedImportType' => 'wxr_file',
- 'isImportTypeSelected' => function() {
- // @TODO Figure out why this function is not hiding the form rows
- $state = wp_interactivity_state();
- $context = wp_interactivity_get_context();
- return $context['importType'] === $state['selectedImportType'];
- },
- ));
- ?>
-
-
-
Data Liberation
-
-
- get_stage();
- $totals = $import_session->get_total_number_of_entities();
- $imported = $import_session->count_imported_entities();
- ?>
-
Current Import
-
get_data_source(); ?>:
- get_human_readable_file_reference(); ?>
-
-
-
- -
- is_stage_completed(WP_Stream_Importer::STAGE_INITIAL)): ?>
- ☑
-
- ☐
-
- New Import Created
-
- -
- is_stage_completed(WP_Stream_Importer::STAGE_INDEX_ENTITIES)): ?>
- ☑
-
- ☐
-
- Index Entities
-
- -
- is_stage_completed(WP_Stream_Importer::STAGE_TOPOLOGICAL_SORT)): ?>
- ☑
-
- ☐
-
- Sort Entities
-
- -
- is_stage_completed(WP_Stream_Importer::STAGE_FRONTLOAD_ASSETS)): ?>
- ☑
-
- ☐
-
- Assets Download
-
- -
- is_stage_completed(WP_Stream_Importer::STAGE_IMPORT_ENTITIES)): ?>
- ☑
-
- ☐
-
- Content Import
-
-
-
-
-
-
New Import Created
-
-
-
Indexing Entities
-
-
-
Sorting Entities
-
Determining optimal import order...
-
-
-
Downloading Assets
-
- get_frontloading_progress();
- if (!empty($frontloading_progress)): ?>
-
-
Downloads in progress:
-
-
-
- | File |
- Progress |
-
-
-
- $progress): ?>
-
- |
- |
-
-
-
-
-
-
Preparing to download assets...
-
-
-
-
Importing Content
-
-
-
- is_stage_completed(WP_Stream_Importer::STAGE_IMPORT_ENTITIES)
- ): ?>
-
-
-
- | Entity |
- Imported |
- Total |
-
-
-
- $count): ?>
-
- |
- |
- |
-
-
-
-
-
-
-
- Import finished!
-
-
- Archive the importing session
-
-
-
- Continue importing
-
-
- Stop importing and archive the session
-
-
-
-
-
-
-
-
-
Previous Import Sessions
-
-
-
-
+
+
+
+ archive();
+ ?>
+
+ get_stage()
+ ) {
+ ?>
+
Last importer output (for debugging):
+
+
+
+ function () {
+ $context = wp_interactivity_get_context();
+ return $context['item']->post_status === 'error';
+ },
+
+ 'isCurrentImportAtStage' => function () {
+ $context = wp_interactivity_get_context();
+ $state = wp_interactivity_state( 'dataLiberation' );
+ return $context['stage']['id'] === $state['currentImport']['stage'];
+ },
+ 'isImportTypeSelected' => function () {
+ $context = wp_interactivity_get_context();
+ $state = wp_interactivity_state( 'dataLiberation' );
+ return $context['importType'] === $state['selectedImportType'];
+ },
+ )
+ )
+ );
+
+ ob_start();
+ include __DIR__ . '/data-liberation-page.php';
+ $html = ob_get_clean();
+ echo wp_interactivity_process_directives( $html );
}
// Handle form submission
-add_action('admin_post_data_liberation_import', function() {
- if (!current_user_can('manage_options')) {
- wp_die('Unauthorized');
- }
-
- // @TODO: check nonce
- // check_admin_nonce('data_liberation_import');
- $data_source = $_POST['data_source'];
- $attachment_id = null;
- $file_name = '';
-
- switch ($data_source) {
- case 'wxr_file':
- if (empty($_FILES['wxr_file']['tmp_name'])) {
- wp_die('Please select a file to upload');
- }
- if (!in_array($_FILES['wxr_file']['type'], ['text/xml', 'application/xml'])) {
- wp_die('Invalid file type');
- }
- /**
- * @TODO: Reconsider storing the file in the media library where everyone
- * can access it via a public URL.
- */
- $attachment_id = media_handle_upload(
- 'wxr_file',
- 0,
- array(),
- array(
- 'mimes' => array(
- 'xml' => 'text/xml',
- 'xml-application' => 'application/xml',
- ),
- // test_form checks:
- // Whether to test that the $_POST['action'] parameter is as expected.
- // It seems useless here and it causes cryptic error "Invalid form submission".
- // Let's just disable it.
- 'test_form' => false,
-
- // @TODO: Find a way to make this type check work.
- 'test_type' => false,
- )
- );
- if (is_wp_error($attachment_id)) {
- wp_die($attachment_id->get_error_message());
- }
- $file_name = $_FILES['wxr_file']['name'];
- $import_session = WP_Import_Session::create(array(
- 'data_source' => 'wxr_file',
- 'attachment_id' => $attachment_id,
- 'file_name' => $file_name,
- ));
- break;
-
- case 'wxr_url':
- if (empty($_POST['wxr_url']) || !filter_var($_POST['wxr_url'], FILTER_VALIDATE_URL)) {
- wp_die('Please enter a valid URL');
- }
- // Don't download the file, it could be 300GB or so. The
- // import callback will stream it as needed.
- $import_session = WP_Import_Session::create(array(
- 'data_source' => 'wxr_url',
- 'source_url' => $_POST['wxr_url'],
- ));
- break;
-
- case 'markdown_zip':
- if (empty($_FILES['markdown_zip']['tmp_name'])) {
- wp_die('Please select a file to upload');
- }
- if ($_FILES['markdown_zip']['type'] !== 'application/zip') {
- wp_die('Invalid file type');
- }
- $attachment_id = media_handle_upload('markdown_zip', 0);
- if (is_wp_error($attachment_id)) {
- wp_die($attachment_id->get_error_message());
- }
- $file_name = $_FILES['markdown_zip']['name'];
- $import_session = WP_Import_Session::create(array(
- 'data_source' => 'markdown_zip',
- 'attachment_id' => $attachment_id,
- 'file_name' => $file_name,
- ));
- break;
-
- default:
- wp_die('Invalid import type');
- }
-
- if ( false === $import_session ) {
- // @TODO: More user friendly error message – maybe redirect back to the import screen and
- // show the error there.
- wp_die('Failed to create an import session');
- }
-
- // Schedule the next import step every minute, so 30 seconds more than the
- // default PHP max_execution_time.
-
- /**
- * @TODO: The schedule doesn't seem to be actually running.
- */
- // if(is_wp_error(wp_schedule_event(time(), 'data_liberation_minute', 'data_liberation_process_import'))) {
- // wp_delete_attachment($attachment_id, true);
- // // @TODO: More user friendly error message – maybe redirect back to the import screen and
- // // show the error there.
- // wp_die('Failed to schedule import – the "data_liberation_minute" schedule may not be registered.');
- // }
-
- wp_redirect(add_query_arg(
- 'message', 'import-scheduled',
- admin_url('admin.php?page=data-liberation')
- ));
- exit;
-});
+add_action(
+ 'admin_post_data_liberation_import',
+ function () {
+ if ( ! current_user_can( 'manage_options' ) ) {
+ wp_die( 'Unauthorized' );
+ }
+
+ // @TODO: check nonce
+ // check_admin_nonce('data_liberation_import');
+ $data_source = $_POST['data_source'];
+ $attachment_id = null;
+ $file_name = '';
+
+ switch ( $data_source ) {
+ case 'wxr_file':
+ if ( empty( $_FILES['wxr_file']['tmp_name'] ) ) {
+ wp_die( 'Please select a file to upload' );
+ }
+ if ( ! in_array( $_FILES['wxr_file']['type'], array( 'text/xml', 'application/xml' ), true ) ) {
+ wp_die( 'Invalid file type' );
+ }
+ /**
+ * @TODO: Reconsider storing the file in the media library where everyone
+ * can access it via a public URL.
+ */
+ $attachment_id = media_handle_upload(
+ 'wxr_file',
+ 0,
+ array(),
+ array(
+ 'mimes' => array(
+ 'xml' => 'text/xml',
+ 'xml-application' => 'application/xml',
+ ),
+ // test_form checks:
+ // Whether to test that the $_POST['action'] parameter is as expected.
+ // It seems useless here and it causes cryptic error "Invalid form submission".
+ // Let's just disable it.
+ 'test_form' => false,
+
+ // @TODO: Find a way to make this type check work.
+ 'test_type' => false,
+ )
+ );
+ if ( is_wp_error( $attachment_id ) ) {
+ wp_die( $attachment_id->get_error_message() );
+ }
+ $file_name = $_FILES['wxr_file']['name'];
+ $import_session = WP_Import_Session::create(
+ array(
+ 'data_source' => 'wxr_file',
+ 'attachment_id' => $attachment_id,
+ 'file_name' => $file_name,
+ )
+ );
+ break;
+
+ case 'wxr_url':
+ if ( empty( $_POST['wxr_url'] ) || ! filter_var( $_POST['wxr_url'], FILTER_VALIDATE_URL ) ) {
+ wp_die( 'Please enter a valid URL' );
+ }
+ // Don't download the file, it could be 300GB or so. The
+ // import callback will stream it as needed.
+ $import_session = WP_Import_Session::create(
+ array(
+ 'data_source' => 'wxr_url',
+ 'source_url' => $_POST['wxr_url'],
+ )
+ );
+ break;
+
+ case 'markdown_zip':
+ if ( empty( $_FILES['markdown_zip']['tmp_name'] ) ) {
+ wp_die( 'Please select a file to upload' );
+ }
+ if ( $_FILES['markdown_zip']['type'] !== 'application/zip' ) {
+ wp_die( 'Invalid file type' );
+ }
+ $attachment_id = media_handle_upload( 'markdown_zip', 0 );
+ if ( is_wp_error( $attachment_id ) ) {
+ wp_die( $attachment_id->get_error_message() );
+ }
+ $file_name = $_FILES['markdown_zip']['name'];
+ $import_session = WP_Import_Session::create(
+ array(
+ 'data_source' => 'markdown_zip',
+ 'attachment_id' => $attachment_id,
+ 'file_name' => $file_name,
+ )
+ );
+ break;
+
+ default:
+ wp_die( 'Invalid import type' );
+ }
+
+ if ( false === $import_session ) {
+ // @TODO: More user friendly error message – maybe redirect back to the import screen and
+ // show the error there.
+ wp_die( 'Failed to create an import session' );
+ }
+
+ // Schedule the next import step every minute, so 30 seconds more than the
+ // default PHP max_execution_time.
+
+ /**
+ * @TODO: The schedule doesn't seem to be actually running.
+ */
+ // if(is_wp_error(wp_schedule_event(time(), 'data_liberation_minute', 'data_liberation_process_import'))) {
+ // wp_delete_attachment($attachment_id, true);
+ // // @TODO: More user friendly error message – maybe redirect back to the import screen and
+ // // show the error there.
+ // wp_die('Failed to schedule import – the "data_liberation_minute" schedule may not be registered.');
+ // }
+
+ wp_redirect(
+ add_query_arg(
+ 'message',
+ 'import-scheduled',
+ admin_url( 'admin.php?page=data-liberation' )
+ )
+ );
+ exit;
+ }
+);
// Process import in the background
function data_liberation_process_import() {
- $session = WP_Import_Session::get_active();
- if (!$session) {
- _doing_it_wrong(
- __METHOD__,
- 'No active import session',
- '1.0.0'
- );
- return false;
- }
- return data_liberation_import_step($session);
+ $session = WP_Import_Session::get_active();
+ if ( ! $session ) {
+ _doing_it_wrong(
+ __METHOD__,
+ 'No active import session',
+ '1.0.0'
+ );
+ return false;
+ }
+ return data_liberation_import_step( $session );
+}
+add_action( 'data_liberation_process_import', 'data_liberation_process_import' );
+
+function data_liberation_import_step( $session ) {
+ $metadata = $session->get_metadata();
+ $importer = data_liberation_create_importer( $metadata );
+ if ( ! $importer ) {
+ return;
+ }
+
+ /**
+ * @TODO: Fix this error we get after a few steps:
+ * Notice: Function WP_XML_Processor::step_in_element was called incorrectly. A tag was not closed. Please see Debugging in WordPress for more information. (This message was added in version WP_VERSION.) in /wordpress/wp-includes/functions.php on line 6114
+ */
+ $soft_time_limit_seconds = 5;
+ $hard_time_limit_seconds = 25;
+ $start_time = microtime( true );
+ $fetched_files = 0;
+ while ( true ) {
+ $time_taken = microtime( true ) - $start_time;
+ if ( $time_taken >= $soft_time_limit_seconds ) {
+ // If we're frontloading and don't have any files fetched yet,
+ // we need to give it more time. Otherwise every time we retry,
+ // we'll start from the beginning and never advance past the
+ // frontloading stage.
+ if ( $importer->get_stage() === WP_Stream_Importer::STAGE_FRONTLOAD_ASSETS ) {
+ if ( $fetched_files > 0 ) {
+ break;
+ }
+ } else {
+ break;
+ }
+ }
+ if ( $time_taken >= $hard_time_limit_seconds ) {
+ // No negotiation, we're done.
+ // @TODO: Make it easily configurable
+ // @TODO: Bump the number of download attempts for the placeholders,
+ // set the status to `error` in each interrupted download.
+ break;
+ }
+
+ if ( true !== $importer->next_step() ) {
+ $session->set_reentrancy_cursor( $importer->get_reentrancy_cursor() );
+
+ $should_advance_to_next_stage = null !== $importer->get_next_stage();
+ if ( $should_advance_to_next_stage ) {
+ if ( WP_Stream_Importer::STAGE_FRONTLOAD_ASSETS === $importer->get_stage() ) {
+ $resolved_all_failures = $session->count_unfinished_frontloading_placeholders() === 0;
+ if ( ! $resolved_all_failures ) {
+ break;
+ }
+ }
+ }
+ if ( ! $importer->advance_to_next_stage() ) {
+ break;
+ }
+ $session->set_stage( $importer->get_stage() );
+ $session->set_reentrancy_cursor( $importer->get_reentrancy_cursor() );
+ continue;
+ }
+
+ switch ( $importer->get_stage() ) {
+ case WP_Stream_Importer::STAGE_INDEX_ENTITIES:
+ // Bump the total number of entities to import.
+ $session->create_frontloading_placeholders( $importer->get_indexed_assets_urls() );
+ $session->bump_total_number_of_entities(
+ $importer->get_indexed_entities_counts()
+ );
+ break;
+ case WP_Stream_Importer::STAGE_FRONTLOAD_ASSETS:
+ $session->bump_frontloading_progress(
+ $importer->get_frontloading_progress(),
+ $importer->get_frontloading_events()
+ );
+ break;
+ case WP_Stream_Importer::STAGE_IMPORT_ENTITIES:
+ $session->bump_imported_entities_counts(
+ $importer->get_imported_entities_counts()
+ );
+ break;
+ }
+
+ $session->set_reentrancy_cursor( $importer->get_reentrancy_cursor() );
+ }
}
-add_action('data_liberation_process_import', 'data_liberation_process_import');
-
-function data_liberation_import_step($session) {
- $metadata = $session->get_metadata();
- $importer = data_liberation_create_importer($metadata);
- if(!$importer) {
- return;
- }
- /**
- * @TODO: Fix this error we get after a few steps:
- * Notice: Function WP_XML_Processor::step_in_element was called incorrectly. A tag was not closed. Please see Debugging in WordPress for more information. (This message was added in version WP_VERSION.) in /wordpress/wp-includes/functions.php on line 6114
- */
-
- // At this moment, the importer knows where to resume from but
- // it hasn't actually pulled the first entity from the stream yet.
- // So let's do that now.
- if($importer->next_step()) {
- // var_dump("Stage: " . $importer->get_stage());
- switch($importer->get_stage()) {
- case WP_Stream_Importer::STAGE_INDEX_ENTITIES:
- // Bump the total number of entities to import.
- var_dump($importer->get_indexed_entities_counts());
- $session->bump_total_number_of_entities([
- ...$importer->get_indexed_entities_counts(),
- 'file' => count($importer->get_indexed_assets_urls())
- ]);
- break;
- case WP_Stream_Importer::STAGE_FRONTLOAD_ASSETS:
- var_dump($importer->get_frontloading_progress());
- var_dump($importer->get_frontloading_events());
- $session->bump_frontloading_progress(
- $importer->get_frontloading_progress(),
- $importer->get_frontloading_events()
- );
- break;
- case WP_Stream_Importer::STAGE_IMPORT_ENTITIES:
- var_dump($importer->get_imported_entities_counts());
- $session->bump_imported_entities_counts(
- $importer->get_imported_entities_counts()
- );
- break;
- }
- }
- // Move to the next step before saving the cursor so that the next
- // import session resumes from the next step.
- if(WP_Stream_Importer::STAGE_FRONTLOAD_ASSETS === $importer->get_stage()) {
- // Define constraints for this run of the frontloading stage.
- // @TODO: Support these constraints at the importer level, not here.
- $min_files_downloaded = 0;
- $soft_time_limit = 10;
- $start_time = time();
- $files_downloaded = 0;
- while(true) {
- if(!$importer->next_step()) {
- break;
- }
- $frontloading_events = $importer->get_frontloading_events();
- foreach($frontloading_events as $event) {
- if($event->type === WP_Attachment_Downloader_Event::SUCCESS) {
- ++$files_downloaded;
- }
- }
- $time_taken = time() - $start_time;
- if($time_taken > $soft_time_limit) {
- if($files_downloaded >= $min_files_downloaded) {
- break;
- }
- }
- }
- } else {
- $importer->next_step();
- }
- if($importer->advance_to_next_stage()) {
- $session->set_stage($importer->get_stage());
- }
- $cursor = $importer->get_reentrancy_cursor();
- if($cursor) {
- $session->set_reentrancy_cursor($cursor);
- }
+
+function data_liberation_create_importer( $import ) {
+ switch ( $import['data_source'] ) {
+ case 'wxr_file':
+ $wxr_path = get_attached_file( $import['attachment_id'] );
+ if ( false === $wxr_path ) {
+ // @TODO: Save the error, report it to the user.
+ return;
+ }
+ $importer = WP_Stream_Importer::create_for_wxr_file(
+ $wxr_path,
+ array(),
+ $import['cursor'] ?? null
+ );
+ break;
+
+ case 'wxr_url':
+ $importer = WP_Stream_Importer::create_for_wxr_url(
+ $import['wxr_url'],
+ array(),
+ $import['cursor'] ?? null
+ );
+ break;
+
+ case 'markdown_zip':
+ // @TODO: Don't unzip. Stream data directly from the ZIP file.
+ $zip_path = get_attached_file( $import['attachment_id'] );
+ $temp_dir = sys_get_temp_dir() . '/data-liberation-markdown-' . $import['attachment_id'];
+ if ( ! file_exists( $temp_dir ) ) {
+ mkdir( $temp_dir, 0777, true );
+ $zip = new ZipArchive();
+ if ( $zip->open( $zip_path ) === true ) {
+ $zip->extractTo( $temp_dir );
+ $zip->close();
+ } else {
+ // @TODO: Save the error, report it to the user
+ return;
+ }
+ }
+ $markdown_root = $temp_dir;
+ $importer = WP_Markdown_Importer::create_for_markdown_directory(
+ $markdown_root,
+ array(
+ 'default_source_site_url' => 'file://' . $markdown_root,
+ 'local_markdown_assets_root' => $markdown_root,
+ 'local_markdown_assets_url_prefix' => '@site/',
+ ),
+ $import['cursor'] ?? null
+ );
+ break;
+ }
+ // @TODO: Consider moving this to the importer constructor.
+ $retries_iterator = new WP_Retry_Frontloading_Iterator( $import['post_id'] );
+ $importer->set_frontloading_retries_iterator( $retries_iterator );
+ return $importer;
}
-function data_liberation_create_importer($import) {
- switch($import['data_source']) {
- case 'wxr_file':
- $wxr_path = get_attached_file($import['attachment_id']);
- if(false === $wxr_path) {
- // @TODO: Save the error, report it to the user.
- return;
- }
- return WP_Stream_Importer::create_for_wxr_file(
- $wxr_path,
- [],
- $import['cursor'] ?? null
- );
-
- case 'wxr_url':
- return WP_Stream_Importer::create_for_wxr_url(
- $import['wxr_url'],
- [],
- $import['cursor'] ?? null
- );
-
- case 'markdown_zip':
- // @TODO: Don't unzip. Stream data directly from the ZIP file.
- $zip_path = get_attached_file($import['attachment_id']);
- $temp_dir = sys_get_temp_dir() . '/data-liberation-markdown-' . $import['attachment_id'];
- if (!file_exists($temp_dir)) {
- mkdir($temp_dir, 0777, true);
- $zip = new ZipArchive();
- if ($zip->open($zip_path) === TRUE) {
- $zip->extractTo($temp_dir);
- $zip->close();
- } else {
- // @TODO: Save the error, report it to the user
- return;
- }
- }
- $markdown_root = $temp_dir;
- return WP_Markdown_Importer::create_for_markdown_directory(
- $markdown_root,
- [
- 'source_site_url' => 'file://' . $markdown_root,
- 'local_markdown_assets_root' => $markdown_root,
- 'local_markdown_assets_url_prefix' => '@site/',
- ],
- $import['cursor'] ?? null
- );
- }
+function data_liberation_get_interactivity_state() {
+ // Populates the initial global state values.
+ $import_history = array_map(
+ function ( $post ) {
+ $import_session = new WP_Import_Session( $post->ID );
+ return array(
+ 'date' => $post->post_date,
+ 'dataSource' => $import_session->get_data_source(),
+ 'timeTaken' => human_time_diff( $import_session->get_started_at(), $import_session->is_finished() ? $import_session->get_finished_at() : time() ),
+ 'entitiesImported' => array_sum( array_column( $import_session->count_imported_entities(), 'imported' ) ),
+ 'totalEntities' => array_sum( array_column( $import_session->count_imported_entities(), 'total' ) ),
+ 'status' => $import_session->get_stage(),
+ );
+ },
+ get_posts(
+ array(
+ 'post_type' => WP_Import_Session::POST_TYPE,
+ 'post_status' => array( 'archived' ),
+ 'posts_per_page' => -1,
+ 'orderby' => 'date',
+ 'order' => 'DESC',
+ )
+ )
+ );
+
+ $import_session = WP_Import_Session::get_active();
+
+ $stages = array();
+ foreach ( WP_Stream_Importer::STAGES_IN_ORDER as $stage ) {
+ $stages[] = array(
+ 'id' => $stage,
+ 'label' => ucfirst( str_replace( '_', ' ', $stage ) ),
+ 'completed' => $import_session ? $import_session->is_stage_completed( $stage ) : false,
+ );
+ }
+
+ $frontloading_progress = array_map(
+ function ( $progress, $url ) {
+ $progress['url'] = $url;
+ return $progress;
+ },
+ $import_session ? $import_session->get_frontloading_progress() : array(),
+ array_keys( $import_session ? $import_session->get_frontloading_progress() : array() )
+ );
+ $frontloading_placeholders = $import_session ? $import_session->get_frontloading_placeholders() : array();
+ return array(
+ // Current import state:
+ 'currentImport' => $import_session
+ ? array(
+ 'active' => true,
+ 'stage' => $import_session->get_stage(),
+ 'dataSource' => $import_session->get_data_source(),
+ 'fileReference' => $import_session->get_human_readable_file_reference(),
+ 'entityCounts' => $import_session->count_imported_entities(),
+ 'frontloadingProgress' => $frontloading_progress,
+ 'hasFrontloadingProgress' => count( $frontloading_progress ) > 0,
+ 'frontloadingPlaceholders' => $frontloading_placeholders,
+ 'hasFrontloadingPlaceholders' => count( $frontloading_placeholders ) > 0,
+ )
+ /**
+ * We need an empty default state that still has all the right data types
+ * to avoid "undefined index" errors in PHP and ".map is not a function"
+ * errors in JS.
+ */
+ : array(
+ 'active' => false,
+ 'stage' => null,
+ 'dataSource' => null,
+ 'fileReference' => null,
+ 'entityCounts' => array(),
+ 'frontloadingProgress' => array(),
+ 'hasFrontloadingProgress' => false,
+ 'frontloadingPlaceholders' => array(),
+ 'hasFrontloadingPlaceholders' => false,
+ ),
+
+ 'stages' => $stages,
+
+ // Import form state:
+ 'selectedImportType' => 'wxr_file',
+
+ // Past imports table:
+ 'importHistory' => array(
+ 'entities' => $import_history,
+ 'numEntities' => count( $import_history ),
+ 'page' => 1,
+ ),
+ );
}
+
+add_action(
+ 'rest_api_init',
+ function () {
+ register_rest_route(
+ 'data-liberation/v1',
+ '/retry-download',
+ array(
+ 'methods' => 'POST',
+ 'callback' => function ( $request ) {
+ $post_id = intval( $request->get_param( 'post_id' ) );
+ $retry_limit = get_post_meta( $post_id, 'retry_limit', true );
+ if ( ! $retry_limit ) {
+ $retry_limit = 3;
+ }
+ update_post_meta( $post_id, 'retry_limit', $retry_limit + 1 );
+
+ return new WP_REST_Response( array( 'success' => true ), 200 );
+ },
+ 'permission_callback' => function () {
+ return current_user_can( 'manage_options' );
+ },
+ )
+ );
+
+ register_rest_route(
+ 'data-liberation/v1',
+ '/ignore-download',
+ array(
+ 'methods' => 'POST',
+ 'callback' => function ( $request ) {
+ $post_id = intval( $request->get_param( 'post_id' ) );
+ wp_update_post(
+ array(
+ 'ID' => $post_id,
+ 'post_status' => WP_Import_Session::FRONTLOAD_STATUS_IGNORED,
+ )
+ );
+
+ return new WP_REST_Response( array( 'success' => true ), 200 );
+ },
+ 'permission_callback' => function () {
+ return current_user_can( 'manage_options' );
+ },
+ )
+ );
+
+ register_rest_route(
+ 'data-liberation/v1',
+ '/change-download-url',
+ array(
+ 'methods' => 'POST',
+ 'callback' => function ( $request ) {
+ $post_id = intval( $request->get_param( 'post_id' ) );
+ $new_url = esc_url_raw( $request->get_param( 'new_url' ) );
+
+ update_post_meta( $post_id, 'current_url', $new_url );
+ update_post_meta( $post_id, 'attempts', 0 );
+ update_post_meta( $post_id, 'retry_limit', 3 );
+
+ return new WP_REST_Response( array( 'success' => true ), 200 );
+ },
+ 'permission_callback' => function () {
+ return current_user_can( 'manage_options' );
+ },
+ )
+ );
+
+ register_rest_route(
+ 'data-liberation/v1',
+ '/interactivity-state',
+ array(
+ 'methods' => 'GET',
+ 'callback' => function () {
+ return new WP_REST_Response( data_liberation_get_interactivity_state(), 200 );
+ },
+ 'permission_callback' => function () {
+ return current_user_can( 'manage_options' );
+ },
+ )
+ );
+ }
+);
diff --git a/packages/playground/data-liberation/project.json b/packages/playground/data-liberation/project.json
index 44c3e7f5a59..8b8bc80e1b4 100644
--- a/packages/playground/data-liberation/project.json
+++ b/packages/playground/data-liberation/project.json
@@ -17,7 +17,7 @@
"options": {
"cwd": "packages/playground/data-liberation",
"commands": [
- "./vendor/bin/phpcs --standard=./phpcs.xml -s ./src"
+ "./vendor/bin/phpcs --standard=./phpcs.xml -s ./src ./*.php"
],
"parallel": false
}
@@ -27,7 +27,7 @@
"options": {
"cwd": "packages/playground/data-liberation",
"commands": [
- "./vendor/bin/phpcbf --standard=./phpcs.xml ./src"
+ "./vendor/bin/phpcbf --standard=./phpcs.xml ./src ./*.php"
],
"parallel": false
}
diff --git a/packages/playground/data-liberation/src/functions.php b/packages/playground/data-liberation/src/functions.php
index de37da5bac2..05baf08aa0e 100644
--- a/packages/playground/data-liberation/src/functions.php
+++ b/packages/playground/data-liberation/src/functions.php
@@ -225,3 +225,12 @@ function data_liberation_import( $path ): bool {
return true;
}
+
+function get_all_post_meta_flat( $post_id ) {
+ return array_map(
+ function ( $value ) {
+ return $value[0];
+ },
+ get_post_meta( $post_id )
+ );
+}
diff --git a/packages/playground/data-liberation/src/import/WP_Attachment_Downloader.php b/packages/playground/data-liberation/src/import/WP_Attachment_Downloader.php
index 261b27c58b4..7dad1f328fe 100644
--- a/packages/playground/data-liberation/src/import/WP_Attachment_Downloader.php
+++ b/packages/playground/data-liberation/src/import/WP_Attachment_Downloader.php
@@ -43,7 +43,7 @@ public function enqueue_if_not_exists( $url, $output_relative_path ) {
if ( file_exists( $output_relative_path ) ) {
$this->pending_events[] = new WP_Attachment_Downloader_Event(
$this->enqueued_url,
- WP_Attachment_Downloader_Event::SUCCESS
+ WP_Attachment_Downloader_Event::ALREADY_EXISTS
);
return true;
}
@@ -70,10 +70,16 @@ public function enqueue_if_not_exists( $url, $output_relative_path ) {
// @TODO: think through the chmod of the created file.
$success = copy( $local_path, $output_relative_path );
- $this->pending_events[] = new WP_Attachment_Downloader_Event(
- $this->enqueued_url,
- $success ? WP_Attachment_Downloader_Event::SUCCESS : WP_Attachment_Downloader_Event::FAILURE
- );
+ $this->pending_events[] = $success
+ ? new WP_Attachment_Downloader_Event(
+ $this->enqueued_url,
+ WP_Attachment_Downloader_Event::SUCCESS
+ )
+ : new WP_Attachment_Downloader_Event(
+ $this->enqueued_url,
+ WP_Attachment_Downloader_Event::FAILURE,
+ 'copy_failed'
+ );
return true;
case 'http':
case 'https':
@@ -147,46 +153,64 @@ public function poll() {
$this->progress[ $original_url ]['received'] += strlen( $chunk );
break;
case Client::EVENT_FAILED:
- if ( isset( $this->fps[ $original_request_id ] ) ) {
- fclose( $this->fps[ $original_request_id ] );
- }
- if ( isset( $this->output_paths[ $original_request_id ] ) ) {
- $partial_file = $this->output_paths[ $original_request_id ] . '.partial';
- if ( file_exists( $partial_file ) ) {
- unlink( $partial_file );
- }
- }
- $this->pending_events[] = new WP_Attachment_Downloader_Event(
- $original_url,
- WP_Attachment_Downloader_Event::FAILURE
- );
- unset( $this->progress[ $original_url ] );
- unset( $this->output_paths[ $original_request_id ] );
+ $this->on_failure( $original_url, $original_request_id, $request->error );
break;
case Client::EVENT_FINISHED:
if ( ! $request->is_redirected() ) {
- // Only clean up if this was the last request in the chain.
- if ( isset( $this->fps[ $original_request_id ] ) ) {
- fclose( $this->fps[ $original_request_id ] );
- }
- if ( isset( $this->output_paths[ $original_request_id ] ) ) {
- if ( false === rename(
- $this->output_paths[ $original_request_id ] . '.partial',
- $this->output_paths[ $original_request_id ]
- ) ) {
- // @TODO: Log an error.
- }
- }
- $this->pending_events[] = new WP_Attachment_Downloader_Event(
- $original_url,
- WP_Attachment_Downloader_Event::SUCCESS
+ // Only process if this was the last request in the chain.
+ $is_success = (
+ $request->response->status_code >= 200 &&
+ $request->response->status_code <= 299
);
- unset( $this->progress[ $original_url ] );
- unset( $this->output_paths[ $original_request_id ] );
+ if ( $is_success ) {
+ $this->on_success( $original_url, $original_request_id );
+ } else {
+ $this->on_failure( $original_url, $original_request_id, 'http_error_' . $request->response->status_code );
+ }
}
break;
}
return true;
}
+
+ private function on_failure( $original_url, $original_request_id, $error = null ) {
+ if ( isset( $this->fps[ $original_request_id ] ) ) {
+ fclose( $this->fps[ $original_request_id ] );
+ }
+ if ( isset( $this->output_paths[ $original_request_id ] ) ) {
+ $partial_file = $this->output_paths[ $original_request_id ] . '.partial';
+ if ( file_exists( $partial_file ) ) {
+ unlink( $partial_file );
+ }
+ }
+ $this->pending_events[] = new WP_Attachment_Downloader_Event(
+ $original_url,
+ WP_Attachment_Downloader_Event::FAILURE,
+ $error
+ );
+ unset( $this->progress[ $original_url ] );
+ unset( $this->output_paths[ $original_request_id ] );
+ }
+
+ private function on_success( $original_url, $original_request_id ) {
+ // Only clean up if this was the last request in the chain.
+ if ( isset( $this->fps[ $original_request_id ] ) ) {
+ fclose( $this->fps[ $original_request_id ] );
+ }
+ if ( isset( $this->output_paths[ $original_request_id ] ) ) {
+ if ( false === rename(
+ $this->output_paths[ $original_request_id ] . '.partial',
+ $this->output_paths[ $original_request_id ]
+ ) ) {
+ // @TODO: Log an error.
+ }
+ }
+ $this->pending_events[] = new WP_Attachment_Downloader_Event(
+ $original_url,
+ WP_Attachment_Downloader_Event::SUCCESS
+ );
+ unset( $this->progress[ $original_url ] );
+ unset( $this->output_paths[ $original_request_id ] );
+ }
}
diff --git a/packages/playground/data-liberation/src/import/WP_Attachment_Downloader_Event.php b/packages/playground/data-liberation/src/import/WP_Attachment_Downloader_Event.php
index b759ebb8fb5..0502107782e 100644
--- a/packages/playground/data-liberation/src/import/WP_Attachment_Downloader_Event.php
+++ b/packages/playground/data-liberation/src/import/WP_Attachment_Downloader_Event.php
@@ -2,14 +2,17 @@
class WP_Attachment_Downloader_Event {
- const SUCCESS = '#success';
- const FAILURE = '#failure';
+ const SUCCESS = '#success';
+ const FAILURE = '#failure';
+ const ALREADY_EXISTS = '#already_exists';
public $type;
public $resource_id;
+ public $error;
- public function __construct( $resource_id, $type ) {
+ public function __construct( $resource_id, $type, $error = null ) {
$this->resource_id = $resource_id;
$this->type = $type;
+ $this->error = $error;
}
}
diff --git a/packages/playground/data-liberation/src/import/WP_Entity_Iterator_Chain.php b/packages/playground/data-liberation/src/import/WP_Entity_Iterator_Chain.php
new file mode 100644
index 00000000000..13128c94f82
--- /dev/null
+++ b/packages/playground/data-liberation/src/import/WP_Entity_Iterator_Chain.php
@@ -0,0 +1,65 @@
+assets_attempts_iterator = $iterator;
+ }
+
+ public function set_entities_iterator( Iterator $iterator ) {
+ $this->entities_iterator = $iterator;
+ }
+
+ public function current(): mixed {
+ $iterator = $this->get_iterator();
+ return $iterator ? $iterator->current() : null;
+ }
+
+ public function key(): mixed {
+ $iterator = $this->get_iterator();
+ return $iterator ? $iterator->key() : null;
+ }
+
+ public function next(): void {
+ $iterator = $this->get_iterator();
+ if ( ! $iterator ) {
+ return;
+ }
+
+ $iterator->next();
+ }
+
+ public function rewind(): void {
+ $iterator = $this->get_iterator();
+ if ( ! $iterator ) {
+ return;
+ }
+ $iterator->rewind();
+ }
+
+ public function valid(): bool {
+ $iterator = $this->get_iterator();
+ return $iterator && $iterator->valid();
+ }
+
+ private function get_iterator(): ?Iterator {
+ if ( $this->assets_attempts_iterator && $this->assets_attempts_iterator->valid() ) {
+ return $this->assets_attempts_iterator;
+ } elseif ( $this->entities_iterator && $this->entities_iterator->valid() ) {
+ return $this->entities_iterator;
+ }
+ return null;
+ }
+
+ public function get_reentrancy_cursor(): mixed {
+ if ( ! $this->entities_iterator ) {
+ return null;
+ }
+ if ( ! method_exists( $this->entities_iterator, 'get_reentrancy_cursor' ) ) {
+ return null;
+ }
+ return $this->entities_iterator->get_reentrancy_cursor();
+ }
+}
diff --git a/packages/playground/data-liberation/src/import/WP_Import_Session.php b/packages/playground/data-liberation/src/import/WP_Import_Session.php
index 13735f58a10..2f56040563f 100644
--- a/packages/playground/data-liberation/src/import/WP_Import_Session.php
+++ b/packages/playground/data-liberation/src/import/WP_Import_Session.php
@@ -8,11 +8,12 @@
*/
class WP_Import_Session {
const POST_TYPE = 'import_session';
+
/**
* @TODO: Make it extendable
* @TODO: Reuse the same entities list as WP_Stream_Importer
*/
- const PROGRESS_ENTITIES = array(
+ const PROGRESS_ENTITIES = array(
'site_option',
'user',
'category',
@@ -22,8 +23,11 @@ class WP_Import_Session {
'post_meta',
'comment',
'comment_meta',
- 'file',
);
+ const FRONTLOAD_STATUS_AWAITING_DOWNLOAD = 'awaiting_download';
+ const FRONTLOAD_STATUS_IGNORED = 'ignored';
+ const FRONTLOAD_STATUS_ERROR = 'error';
+ const FRONTLOAD_STATUS_SUCCEEDED = 'succeeded';
private $post_id;
private $cached_stage;
@@ -175,6 +179,7 @@ public function get_id() {
public function get_metadata() {
$cursor = $this->get_reentrancy_cursor();
return array(
+ 'post_id' => $this->post_id,
'cursor' => $cursor ? $cursor : null,
'data_source' => get_post_meta( $this->post_id, 'data_source', true ),
'source_url' => get_post_meta( $this->post_id, 'source_url', true ),
@@ -214,7 +219,11 @@ public function archive() {
public function count_imported_entities() {
$progress = array();
foreach ( self::PROGRESS_ENTITIES as $entity ) {
- $progress[ $entity ] = (int) get_post_meta( $this->post_id, 'imported_' . $entity, true );
+ $progress[] = array(
+ 'label' => $entity,
+ 'imported' => (int) get_post_meta( $this->post_id, 'imported_' . $entity, true ),
+ 'total' => (int) get_post_meta( $this->post_id, 'total_' . $entity, true ),
+ );
}
return $progress;
}
@@ -267,13 +276,154 @@ public function bump_imported_entities_counts( $newly_imported_entities ) {
}
}
+ public function count_unfinished_frontloading_placeholders() {
+ global $wpdb;
+ return (int) $wpdb->get_var(
+ $wpdb->prepare(
+ "SELECT COUNT(*) FROM $wpdb->posts
+ WHERE post_type = 'frontloading_placeholder'
+ AND post_parent = %d
+ AND post_status != %s
+ AND post_status != %s",
+ $this->post_id,
+ self::FRONTLOAD_STATUS_SUCCEEDED,
+ self::FRONTLOAD_STATUS_IGNORED
+ )
+ );
+ }
+
+ public function get_frontloading_placeholders( $options = array() ) {
+ $query = new WP_Query(
+ array(
+ 'post_type' => 'frontloading_placeholder',
+ 'post_status' => 'any',
+ 'post_parent' => $this->post_id,
+ 'posts_per_page' => $options['per_page'] ?? 25,
+ 'paged' => $options['page'] ?? 1,
+ 'orderby' => array(
+ 'post_status' => array(
+ self::FRONTLOAD_STATUS_ERROR => 0,
+ self::FRONTLOAD_STATUS_AWAITING_DOWNLOAD => 1,
+ 'any' => 2,
+ ),
+ 'ID' => 'ASC',
+ ),
+ )
+ );
+
+ if ( ! $query->have_posts() ) {
+ return array();
+ }
+
+ $posts = $query->posts;
+ $ids = array_map(
+ function ( $post ) {
+ return $post->ID;
+ },
+ $posts
+ );
+ update_meta_cache( 'post', $ids );
+ foreach ( $posts as $post ) {
+ $post->meta = get_all_post_meta_flat( $post->ID );
+ }
+
+ return $posts;
+ }
+
public function get_total_number_of_entities() {
$totals = array();
foreach ( static::PROGRESS_ENTITIES as $field ) {
$totals[ $field ] = (int) get_post_meta( $this->post_id, 'total_' . $field, true );
}
+ $totals['download'] = $this->get_total_number_of_assets();
return $totals;
}
+
+ public function get_total_number_of_assets() {
+ global $wpdb;
+ return (int) $wpdb->get_var(
+ $wpdb->prepare(
+ "SELECT COUNT(*) FROM $wpdb->posts
+ WHERE post_type = 'frontloading_placeholder'
+ AND post_parent = %d",
+ $this->post_id
+ )
+ );
+ }
+
+ public function get_frontloading_placeholder( $url ) {
+ global $wpdb;
+ $id = $wpdb->get_var(
+ $wpdb->prepare(
+ "SELECT p.ID FROM $wpdb->posts p
+ INNER JOIN $wpdb->postmeta pm ON p.ID = pm.post_id
+ WHERE p.post_type = 'frontloading_placeholder'
+ AND p.post_parent = %d
+ AND pm.meta_key = 'current_url'
+ AND pm.meta_value = %s
+ LIMIT 1",
+ $this->post_id,
+ $url
+ )
+ );
+ return get_post( $id );
+ }
+
+ /**
+ * Creates placeholder attachments for the assets to be downloaded in the
+ * frontloading stage.
+ */
+ public function create_frontloading_placeholders( $urls ) {
+ global $wpdb;
+
+ foreach ( $urls as $url => $_ ) {
+ /**
+ * Check if placeholder with this URL already exists
+ * There's a race condition here – another insert may happen
+ * between the check and the insert.
+ * @TODO: Explore solutions. A custom table with a UNIQUE constraint
+ * may or may not be an option, depending on the performance impact
+ * on 100GB+ VIP databases.
+ */
+ $exists = $wpdb->get_var(
+ $wpdb->prepare(
+ "SELECT ID FROM $wpdb->posts
+ WHERE post_type = 'frontloading_placeholder'
+ AND post_parent = %d
+ AND guid = %s
+ LIMIT 1",
+ $this->post_id,
+ $url
+ )
+ );
+
+ if ( $exists ) {
+ continue;
+ }
+
+ $post_data = array(
+ 'post_type' => 'frontloading_placeholder',
+ 'post_parent' => $this->post_id,
+ 'post_title' => basename( $url ),
+ 'post_status' => self::FRONTLOAD_STATUS_AWAITING_DOWNLOAD,
+ 'guid' => $url,
+ 'meta_input' => array(
+ 'original_url' => $url,
+ 'current_url' => $url,
+ 'attempts' => 0,
+ 'last_error' => null,
+ 'target_path' => '',
+ ),
+ );
+ $insertion_result = wp_insert_post( $post_data );
+ if ( is_wp_error( $insertion_result ) ) {
+ // @TODO: How to handle an insertion failure?
+ throw new Exception( 'Failed to insert frontloading placeholder' );
+ return false;
+ }
+ }
+ }
+
/**
* Sets the total number of entities to import for each type.
*
@@ -314,26 +464,54 @@ public function bump_total_number_of_entities( $newly_indexed_entities ) {
public function bump_frontloading_progress( $frontloading_progress, $events = array() ) {
update_post_meta( $this->post_id, 'frontloading_progress', $frontloading_progress );
- $successes = 0;
foreach ( $events as $event ) {
- if ( $event->type === WP_Attachment_Downloader_Event::SUCCESS ) {
- ++$successes;
- } else {
- // @TODO: Store error.
+ $url = $event->resource_id;
+ $placeholder = $this->get_frontloading_placeholder( $url );
+ if ( ! $placeholder ) {
+ _doing_it_wrong(
+ __METHOD__,
+ 'Frontloading placeholder post not found for URL: ' . $url,
+ '1.0.0'
+ );
+ continue;
+ }
+
+ update_post_meta( $placeholder->ID, 'last_error', $event->error );
+
+ $attempts = get_post_meta( $placeholder->ID, 'attempts', true );
+ $new_attempts = $attempts;
+ switch ( $event->type ) {
+ case WP_Attachment_Downloader_Event::SUCCESS:
+ $new_status = self::FRONTLOAD_STATUS_SUCCEEDED;
+ $new_attempts = $attempts + 1;
+ break;
+ case WP_Attachment_Downloader_Event::ALREADY_EXISTS:
+ $new_status = self::FRONTLOAD_STATUS_SUCCEEDED;
+ break;
+ case WP_Attachment_Downloader_Event::FAILURE:
+ $new_status = self::FRONTLOAD_STATUS_ERROR;
+ $new_attempts = $attempts + 1;
+ break;
+ }
+
+ if ( $new_attempts !== $attempts ) {
+ update_post_meta( $placeholder->ID, 'attempts', $new_attempts );
+ }
+
+ if ( $new_status !== $placeholder->post_status ) {
+ wp_update_post(
+ array(
+ 'ID' => $placeholder->ID,
+ 'post_status' => $new_status,
+ )
+ );
}
- }
- if ( $successes > 0 ) {
- // @TODO: Consider not treating files as a special case.
- $this->bump_imported_entities_counts(
- array(
- 'file' => $successes,
- )
- );
}
}
public function get_frontloading_progress() {
- return get_post_meta( $this->post_id, 'frontloading_progress', true ) ?? array();
+ $meta = get_post_meta( $this->post_id, 'frontloading_progress', true );
+ return $meta ? $meta : array();
}
public function is_stage_completed( $stage ) {
diff --git a/packages/playground/data-liberation/src/import/WP_Markdown_Importer.php b/packages/playground/data-liberation/src/import/WP_Markdown_Importer.php
index e76e6d643f3..1acc69e5434 100644
--- a/packages/playground/data-liberation/src/import/WP_Markdown_Importer.php
+++ b/packages/playground/data-liberation/src/import/WP_Markdown_Importer.php
@@ -5,7 +5,7 @@ class WP_Markdown_Importer extends WP_Stream_Importer {
public static function create_for_markdown_directory( $markdown_directory, $options = array(), $cursor = null ) {
return static::create(
function ( $cursor = null ) use ( $markdown_directory ) {
- return WP_Markdown_Importer::create( $markdown_directory, $cursor );
+ return new WP_Markdown_Directory_Tree_Reader( $markdown_directory, $cursor );
},
$options,
$cursor
diff --git a/packages/playground/data-liberation/src/import/WP_Retry_Frontloading_Iterator.php b/packages/playground/data-liberation/src/import/WP_Retry_Frontloading_Iterator.php
new file mode 100644
index 00000000000..d73357b14ca
--- /dev/null
+++ b/packages/playground/data-liberation/src/import/WP_Retry_Frontloading_Iterator.php
@@ -0,0 +1,98 @@
+import_post_id = $import_post_id;
+ }
+
+ public function current(): mixed {
+ if ( ! $this->current ) {
+ return null;
+ }
+ return new WP_Imported_Entity(
+ 'asset_retry',
+ array(
+ 'current_url' => $this->current->meta['current_url'],
+ 'original_url' => $this->current->meta['original_url'],
+ )
+ );
+ }
+
+ public function next(): void {
+ if ( empty( $this->placeholders ) ) {
+ $this->query_next_page();
+ }
+ $this->current = array_shift( $this->placeholders );
+ $this->rewound = false;
+ }
+
+ public function key(): mixed {
+ if ( ! $this->current ) {
+ return null;
+ }
+ return $this->current->ID;
+ }
+
+ public function valid(): bool {
+ if ( $this->rewound ) {
+ $this->next();
+ }
+ return null !== $this->current;
+ }
+
+ public function rewind(): void {
+ $this->placeholders = array();
+ $this->last_id_on_page = null;
+ $this->rewound = true;
+ }
+
+ private function query_next_page() {
+ global $wpdb;
+
+ $where_clauses = array(
+ $wpdb->prepare( 'post_type = %s', 'frontloading_placeholder' ),
+ $wpdb->prepare( 'post_parent = %d', $this->import_post_id ),
+ $wpdb->prepare( 'post_status = %s', WP_Import_Session::FRONTLOAD_STATUS_ERROR ),
+ "pm.meta_key = 'attempts'",
+ 'pm.meta_value < 3',
+ );
+
+ if ( null !== $this->last_id_on_page ) {
+ $where_clauses[] = $wpdb->prepare( 'p.ID > %d', $this->last_id_on_page );
+ }
+
+ $where = implode( ' AND ', $where_clauses );
+ $this->placeholders = $wpdb->get_results(
+ "SELECT p.* FROM {$wpdb->posts} p
+ INNER JOIN {$wpdb->postmeta} pm ON p.ID = pm.post_id
+ WHERE {$where}
+ ORDER BY p.ID ASC
+ LIMIT 100"
+ );
+ $last_placeholder = end( $this->placeholders );
+ if ( $last_placeholder ) {
+ $this->last_id_on_page = $last_placeholder->ID;
+ }
+
+ update_meta_cache(
+ 'post',
+ array_map(
+ function ( $placeholder ) {
+ return $placeholder->ID;
+ },
+ $this->placeholders
+ )
+ );
+
+ foreach ( $this->placeholders as $placeholder ) {
+ $placeholder->meta = get_all_post_meta_flat( $placeholder->ID );
+ }
+ }
+}
diff --git a/packages/playground/data-liberation/src/import/WP_Stream_Importer.php b/packages/playground/data-liberation/src/import/WP_Stream_Importer.php
index 03e45800a76..05b85ff05b0 100644
--- a/packages/playground/data-liberation/src/import/WP_Stream_Importer.php
+++ b/packages/playground/data-liberation/src/import/WP_Stream_Importer.php
@@ -14,8 +14,10 @@
*
* @TODO:
* ✅ Re-entrant import
- * * Log errors for the user to decide what to do.
- * * Log progress information.
+ * * Log progress and errors for the user to decide what to do.
+ * * Indexing
+ * * ✅ Frontloading
+ * * Entity import
* * Error out if `source_site_url` is not set by the time we're processing
* the first encountered URL.
* * Disable anything remotely related to KSES during the import. KSES
@@ -133,7 +135,11 @@ private function initialize_from_cursor( $cursor ) {
return false;
}
$this->stage = $cursor['stage'];
+ $this->next_stage = $cursor['next_stage'];
$this->resume_at_entity = $cursor['resume_at_entity'];
+ if ( ! empty( $cursor['source_site_url'] ) ) {
+ $this->source_site_url = $cursor['source_site_url'];
+ }
return true;
}
@@ -141,7 +147,13 @@ public function get_reentrancy_cursor() {
return json_encode(
array(
'stage' => $this->stage,
+ /**
+ * Store `next_stage` to distinguish between the start and the end of the entity
+ * stream. `resume_at_entity` may be null in both cases.
+ */
+ 'next_stage' => $this->next_stage,
'resume_at_entity' => $this->resume_at_entity,
+ 'source_site_url' => $this->source_site_url,
)
);
}
@@ -172,11 +184,16 @@ private function __construct(
) {
$this->entity_iterator_factory = $entity_iterator_factory;
$this->options = $options;
- if ( isset( $options['source_site_url'] ) ) {
- $this->source_site_url = $options['source_site_url'];
+ if ( isset( $options['default_source_site_url'] ) ) {
+ $this->source_site_url = $options['default_source_site_url'];
}
}
+ private $frontloading_retries_iterator;
+ public function set_frontloading_retries_iterator( $frontloading_retries_iterator ) {
+ $this->frontloading_retries_iterator = $frontloading_retries_iterator;
+ }
+
/**
* The WordPress entity importer instance.
* @TODO: Consider inlining the importer code into this class.
@@ -186,9 +203,6 @@ private function __construct(
private $importer;
public function next_step() {
- if ( null !== $this->next_stage ) {
- return false;
- }
switch ( $this->stage ) {
case self::STAGE_INITIAL:
$this->next_stage = self::STAGE_INDEX_ENTITIES;
@@ -241,6 +255,10 @@ public function advance_to_next_stage() {
private $indexed_assets_urls = array();
private function index_next_entities( $count = 10000 ) {
+ if ( null !== $this->next_stage ) {
+ return false;
+ }
+
if ( null === $this->entity_iterator ) {
$this->entity_iterator = $this->create_entity_iterator();
}
@@ -290,22 +308,29 @@ private function index_next_entities( $count = 10000 ) {
* every single post in memory for sure, otherwise WordPress would not
* be able to render it, but can we hold all the URLs from 10k posts at once?
*/
- if ( 'post' === $type ) {
- $data = $entity->get_data();
- if ( isset( $data['post_type'] ) && $data['post_type'] === 'attachment' ) {
- // @TODO: Consider using sha1 hashes to prevent huge URLs from blowing up the memory.
- $this->indexed_assets_urls[ $data['attachment_url'] ] = true;
- } elseif ( isset( $data['post_content'] ) ) {
- $post = $data;
- $p = new WP_Block_Markup_Url_Processor( $post['post_content'], $this->source_site_url );
- while ( $p->next_url() ) {
- if ( ! $this->url_processor_matched_asset_url( $p ) ) {
- continue;
- }
+ $data = $entity->get_data();
+ switch ( $type ) {
+ case 'site_option':
+ if ( $data['option_name'] === 'home' ) {
+ $this->source_site_url = $data['option_value'];
+ }
+ break;
+ case 'post':
+ if ( isset( $data['post_type'] ) && $data['post_type'] === 'attachment' ) {
// @TODO: Consider using sha1 hashes to prevent huge URLs from blowing up the memory.
- $this->indexed_assets_urls[ $p->get_raw_url() ] = true;
+ $this->indexed_assets_urls[ $data['attachment_url'] ] = true;
+ } elseif ( isset( $data['post_content'] ) ) {
+ $post = $data;
+ $p = new WP_Block_Markup_Url_Processor( $post['post_content'], $this->source_site_url );
+ while ( $p->next_url() ) {
+ if ( ! $this->url_processor_matched_asset_url( $p ) ) {
+ continue;
+ }
+ // @TODO: Consider using sha1 hashes to prevent huge URLs from blowing up the memory.
+ $this->indexed_assets_urls[ $p->get_raw_url() ] = true;
+ }
}
- }
+ break;
}
$this->entity_iterator->next();
@@ -347,8 +372,9 @@ private function frontloading_advance_reentrancy_cursor() {
while ( $this->downloader->next_event() ) {
$event = $this->downloader->get_event();
switch ( $event->type ) {
- case WP_Attachment_Downloader_Event::SUCCESS:
case WP_Attachment_Downloader_Event::FAILURE:
+ case WP_Attachment_Downloader_Event::SUCCESS:
+ case WP_Attachment_Downloader_Event::ALREADY_EXISTS:
$this->frontloading_events[] = $event;
foreach ( array_keys( $this->active_downloads ) as $entity_cursor ) {
unset( $this->active_downloads[ $entity_cursor ][ $event->resource_id ] );
@@ -378,8 +404,14 @@ private function frontloading_advance_reentrancy_cursor() {
*/
private function frontload_next_entity() {
if ( null === $this->entity_iterator ) {
- $this->entity_iterator = $this->create_entity_iterator();
- $this->downloader = new WP_Attachment_Downloader( $this->options['uploads_path'] );
+ $this->entity_iterator = new WP_Entity_Iterator_Chain();
+ if ( null !== $this->frontloading_retries_iterator ) {
+ $this->entity_iterator->set_assets_attempts_iterator( $this->frontloading_retries_iterator );
+ }
+ if ( null === $this->next_stage ) {
+ $this->entity_iterator->set_entities_iterator( $this->create_entity_iterator() );
+ }
+ $this->downloader = new WP_Attachment_Downloader( $this->options['uploads_path'] );
}
// Clear the frontloading events from the previous pass.
@@ -403,6 +435,7 @@ private function frontload_next_entity() {
* * For large files, every time a full megabyte is downloaded above 10MB.
*/
if ( true === $this->downloader->poll() ) {
+ $this->frontloading_advance_reentrancy_cursor();
return true;
}
}
@@ -411,7 +444,7 @@ private function frontload_next_entity() {
if ( ! $this->entity_iterator->valid() && ! $this->downloader->has_pending_requests() ) {
// This is an assertion to make double sure we're emptying the state queue.
if ( ! empty( $this->active_downloads ) ) {
- _doing_it_wrong( __METHOD__, 'Frontloading queue is not empty.', '1.0' );
+ _doing_it_wrong( __METHOD__, '$active_downloads queue was not empty at the end of the frontloading stage.', '1.0' );
}
$this->downloader = null;
$this->active_downloads = array();
@@ -431,14 +464,17 @@ private function frontload_next_entity() {
$data = $entity->get_data();
switch ( $entity->get_type() ) {
- case 'site_option':
- if ( $data['option_name'] === 'home' ) {
- $this->source_site_url = $data['option_value'];
- }
+ case 'asset_retry':
+ $this->enqueue_attachment_download(
+ $data['current_url'],
+ array(
+ 'original_url' => $data['original_url'],
+ )
+ );
break;
case 'post':
if ( isset( $data['post_type'] ) && $data['post_type'] === 'attachment' ) {
- $this->enqueue_attachment_download( $data['attachment_url'], null );
+ $this->enqueue_attachment_download( $data['attachment_url'] );
} elseif ( isset( $data['post_content'] ) ) {
$post = $data;
$p = new WP_Block_Markup_Url_Processor( $post['post_content'], $this->source_site_url );
@@ -448,18 +484,15 @@ private function frontload_next_entity() {
}
$this->enqueue_attachment_download(
$p->get_raw_url(),
- $post['source_path'] ?? $post['slug'] ?? null
+ array(
+ 'context_path' => $post['source_path'] ?? $post['slug'] ?? null,
+ )
);
}
}
break;
}
- /**
- * @TODO: Update the progress information.
- * @TODO: Save the freshly requested URLs to the cursor.
- */
-
// Move on to the next entity.
$this->entity_iterator->next();
@@ -475,6 +508,10 @@ private function frontload_next_entity() {
* the API consumer?
*/
private function import_next_entity() {
+ if ( null !== $this->next_stage ) {
+ return false;
+ }
+
$this->imported_entities_counts = array();
if ( null === $this->entity_iterator ) {
@@ -567,9 +604,11 @@ public function get_imported_entities_counts() {
return $this->imported_entities_counts;
}
- private function enqueue_attachment_download( string $raw_url, $context_path = null ) {
+ private function enqueue_attachment_download( string $raw_url, $options = array() ) {
+ $context_path = $options['context_path'] ?? null;
+ $original_url = $options['original_url'] ?? $raw_url;
$url = $this->rewrite_attachment_url( $raw_url, $context_path );
- $asset_filename = $this->new_asset_filename( $raw_url );
+ $asset_filename = $this->new_asset_filename( $original_url );
$output_filename = ltrim( $asset_filename, '/' );
$enqueued = $this->downloader->enqueue_if_not_exists( $url, $output_filename );
diff --git a/packages/playground/data-liberation/src/wxr/WP_WXR_Reader.php b/packages/playground/data-liberation/src/wxr/WP_WXR_Reader.php
index 939fff254da..25c21ff6088 100644
--- a/packages/playground/data-liberation/src/wxr/WP_WXR_Reader.php
+++ b/packages/playground/data-liberation/src/wxr/WP_WXR_Reader.php
@@ -925,6 +925,10 @@ public function valid(): bool {
}
public function rewind(): void {
+ // Haven't started yet.
+ if ( null === $this->last_next_result ) {
+ return;
+ }
_doing_it_wrong(
__METHOD__,
'WP_WXR_Reader does not support rewinding.',
diff --git a/packages/playground/data-liberation/tests/wxr/frontloading-attachments-post-contents-and-404s.xml b/packages/playground/data-liberation/tests/wxr/frontloading-attachments-post-contents-and-404s.xml
new file mode 100644
index 00000000000..a6d73f0e1c8
--- /dev/null
+++ b/packages/playground/data-liberation/tests/wxr/frontloading-attachments-post-contents-and-404s.xml
@@ -0,0 +1,116 @@
+
+
+
+
+
+Theme Accessibility Unit Test Data
+http://wpthemetestdata.wordpress.com
+Just another WordPress website with a purposefully really long description
+Tue, 02 Sep 2014 17:16:26 +0000
+en
+1.2
+http://wordpress.com/
+http://wpthemetestdata.wordpress.com
+
+ a11yteam
+ a11yteam@wordpress.org
+
+
+
+
+-
+ canola2
+ http://wpthemetestdata.wordpress.com/2010/09/10/post-format-gallery/canola2/
+ Mon, 10 Jan 2011 13:17:54 +0000
+ a11yteam
+ http://wpthemetestdata.files.wordpress.com/2008/06/canola2.jpg
+
+
+
+ 611
+ 2011-01-10 06:17:54
+ 2011-01-10 13:17:54
+ open
+ closed
+ canola2
+ inherit
+ 555
+ 0
+ attachment
+
+ 0
+ https://wpthemetestdata.files.wordpress.com/2008/06/canola2.jpg
+
+ _wp_attachment_image_alt
+
+
+
+ _wp_attachment_image_alt
+
+
+
+ _attachment_original_parent_id
+
+
+
+ -
+
+ https://playground.internal/path/?p=1
+ Wed, 05 Jun 2024 16:04:48 +0000
+
+ https://playground.internal/path/?p=1
+
+
+
Two roads diverged in a yellow wood,
And sorry I could not travel both
+
+
+
+
+
+
+
+]]>
+
+ 17973
+
+
+
+
+
+
+
+
+ 0
+ 0
+
+
+ 0
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file