diff --git a/CHANGELOG.md b/CHANGELOG.md index 33b20469135..34c3ff0a6f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,30 +4,29 @@ All notable changes to this project are documented in this file by a CI job that runs on every NPM release. The file follows the [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) format. -## [v1.0.14] (2024-12-02) +## [v1.0.14] (2024-12-02) ### Blueprints -- Resolve the latest WordPress version from the API instead of assuming it's the same as the last minified build. ([#2027](https://github.com/WordPress/wordpress-playground/pull/2027)) +- Resolve the latest WordPress version from the API instead of assuming it's the same as the last minified build. ([#2027](https://github.com/WordPress/wordpress-playground/pull/2027)) ### Tools - #### Blueprints Builder -- Add installPlugin support for single plugin files. ([#2033](https://github.com/WordPress/wordpress-playground/pull/2033)) +- Add installPlugin support for single plugin files. ([#2033](https://github.com/WordPress/wordpress-playground/pull/2033)) ### PHP WebAssembly -- Networking: Preserve the content-type header when fetch()-ing. ([#2028](https://github.com/WordPress/wordpress-playground/pull/2028)) +- Networking: Preserve the content-type header when fetch()-ing. ([#2028](https://github.com/WordPress/wordpress-playground/pull/2028)) ### Website -- [Web] Re-enable wp-cron. ([#2039](https://github.com/WordPress/wordpress-playground/pull/2039)) +- [Web] Re-enable wp-cron. ([#2039](https://github.com/WordPress/wordpress-playground/pull/2039)) ### Various -- [Data Liberation] WP_Stream_Importer: User-driven incremental import. ([#2013](https://github.com/WordPress/wordpress-playground/pull/2013)) +- [Data Liberation] WP_Stream_Importer: User-driven incremental import. ([#2013](https://github.com/WordPress/wordpress-playground/pull/2013)) ### Contributors @@ -35,7 +34,6 @@ The following contributors merged PRs in this release: @adamziel @brandonpayton - ## [v1.0.13] (2024-11-25) ### Enhancements diff --git a/packages/docs/site/docs/main/changelog.md b/packages/docs/site/docs/main/changelog.md index aef43b75000..53f538b8aa3 100644 --- a/packages/docs/site/docs/main/changelog.md +++ b/packages/docs/site/docs/main/changelog.md @@ -9,30 +9,29 @@ All notable changes to this project are documented in this file by a CI job that runs on every NPM release. The file follows the [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) format. -## [v1.0.14] (2024-12-02) +## [v1.0.14] (2024-12-02) ### Blueprints -- Resolve the latest WordPress version from the API instead of assuming it's the same as the last minified build. ([#2027](https://github.com/WordPress/wordpress-playground/pull/2027)) +- Resolve the latest WordPress version from the API instead of assuming it's the same as the last minified build. ([#2027](https://github.com/WordPress/wordpress-playground/pull/2027)) ### Tools - #### Blueprints Builder -- Add installPlugin support for single plugin files. ([#2033](https://github.com/WordPress/wordpress-playground/pull/2033)) +- Add installPlugin support for single plugin files. ([#2033](https://github.com/WordPress/wordpress-playground/pull/2033)) ### PHP WebAssembly -- Networking: Preserve the content-type header when fetch()-ing. ([#2028](https://github.com/WordPress/wordpress-playground/pull/2028)) +- Networking: Preserve the content-type header when fetch()-ing. ([#2028](https://github.com/WordPress/wordpress-playground/pull/2028)) ### Website -- [Web] Re-enable wp-cron. ([#2039](https://github.com/WordPress/wordpress-playground/pull/2039)) +- [Web] Re-enable wp-cron. ([#2039](https://github.com/WordPress/wordpress-playground/pull/2039)) ### Various -- [Data Liberation] WP_Stream_Importer: User-driven incremental import. ([#2013](https://github.com/WordPress/wordpress-playground/pull/2013)) +- [Data Liberation] WP_Stream_Importer: User-driven incremental import. ([#2013](https://github.com/WordPress/wordpress-playground/pull/2013)) ### Contributors @@ -40,7 +39,6 @@ The following contributors merged PRs in this release: @adamziel @brandonpayton - ## [v1.0.13] (2024-11-25) ### Enhancements diff --git a/packages/php-wasm/compile/php/Dockerfile b/packages/php-wasm/compile/php/Dockerfile index 1c4c9f94dcf..0c5bef72150 100644 --- a/packages/php-wasm/compile/php/Dockerfile +++ b/packages/php-wasm/compile/php/Dockerfile @@ -569,6 +569,7 @@ export ASYNCIFY_ONLY=$'"rc_dtor_func",\ "_php_stream_fill_read_buffer",\ "_php_stream_read",\ "php_stream_read_to_str",\ +"php_userstreamop_read",\ "zif_fread",\ "wasm_read",\ "php_stdiop_read",\ diff --git a/packages/php-wasm/node/asyncify/8_3_0/php_8_3.wasm b/packages/php-wasm/node/asyncify/8_3_0/php_8_3.wasm index 8b8b0c94d69..5c35cb2d32f 100755 Binary files a/packages/php-wasm/node/asyncify/8_3_0/php_8_3.wasm and b/packages/php-wasm/node/asyncify/8_3_0/php_8_3.wasm differ diff --git a/packages/php-wasm/node/asyncify/php_8_3.js b/packages/php-wasm/node/asyncify/php_8_3.js index fdd708b4594..8d6e96343e1 100644 --- a/packages/php-wasm/node/asyncify/php_8_3.js +++ b/packages/php-wasm/node/asyncify/php_8_3.js @@ -1,6 +1,6 @@ const dependencyFilename = __dirname + '/8_3_0/php_8_3.wasm'; export { dependencyFilename }; -export const dependenciesTotalSize = 15083501; +export const dependenciesTotalSize = 15085661; export function init(RuntimeName, PHPLoader) { /** * Overrides Emscripten's default ExitStatus object which gets diff --git a/packages/playground/data-liberation/bootstrap.php b/packages/playground/data-liberation/bootstrap.php index 2bd3420eaa1..8e4e5f0177e 100644 --- a/packages/playground/data-liberation/bootstrap.php +++ b/packages/playground/data-liberation/bootstrap.php @@ -17,25 +17,25 @@ require_once __DIR__ . '/src/byte-readers/WP_Remote_File_Reader.php'; require_once __DIR__ . '/src/byte-readers/WP_Remote_File_Ranged_Reader.php'; -if(!class_exists('WP_HTML_Tag_Processor')) { - require_once __DIR__ . "/src/wordpress-core-html-api/class-wp-html-token.php"; - require_once __DIR__ . "/src/wordpress-core-html-api/class-wp-html-span.php"; - require_once __DIR__ . "/src/wordpress-core-html-api/class-wp-html-text-replacement.php"; - require_once __DIR__ . "/src/wordpress-core-html-api/class-wp-html-decoder.php"; - require_once __DIR__ . "/src/wordpress-core-html-api/class-wp-html-attribute-token.php"; - - require_once __DIR__ . "/src/wordpress-core-html-api/class-wp-html-decoder.php"; - require_once __DIR__ . "/src/wordpress-core-html-api/class-wp-html-tag-processor.php"; - require_once __DIR__ . "/src/wordpress-core-html-api/class-wp-html-open-elements.php"; - require_once __DIR__ . "/src/wordpress-core-html-api/class-wp-token-map.php"; - require_once __DIR__ . "/src/wordpress-core-html-api/html5-named-character-references.php"; - require_once __DIR__ . "/src/wordpress-core-html-api/class-wp-html-active-formatting-elements.php"; - require_once __DIR__ . "/src/wordpress-core-html-api/class-wp-html-processor-state.php"; - require_once __DIR__ . "/src/wordpress-core-html-api/class-wp-html-unsupported-exception.php"; - require_once __DIR__ . "/src/wordpress-core-html-api/class-wp-html-processor.php"; +if ( ! class_exists( 'WP_HTML_Tag_Processor' ) ) { + require_once __DIR__ . '/src/wordpress-core-html-api/class-wp-html-token.php'; + require_once __DIR__ . '/src/wordpress-core-html-api/class-wp-html-span.php'; + require_once __DIR__ . '/src/wordpress-core-html-api/class-wp-html-text-replacement.php'; + require_once __DIR__ . '/src/wordpress-core-html-api/class-wp-html-decoder.php'; + require_once __DIR__ . '/src/wordpress-core-html-api/class-wp-html-attribute-token.php'; + + require_once __DIR__ . '/src/wordpress-core-html-api/class-wp-html-decoder.php'; + require_once __DIR__ . '/src/wordpress-core-html-api/class-wp-html-tag-processor.php'; + require_once __DIR__ . '/src/wordpress-core-html-api/class-wp-html-open-elements.php'; + require_once __DIR__ . '/src/wordpress-core-html-api/class-wp-token-map.php'; + require_once __DIR__ . '/src/wordpress-core-html-api/html5-named-character-references.php'; + require_once __DIR__ . '/src/wordpress-core-html-api/class-wp-html-active-formatting-elements.php'; + require_once __DIR__ . '/src/wordpress-core-html-api/class-wp-html-processor-state.php'; + require_once __DIR__ . '/src/wordpress-core-html-api/class-wp-html-unsupported-exception.php'; + require_once __DIR__ . '/src/wordpress-core-html-api/class-wp-html-processor.php'; } -if (!isset($html5_named_character_references)) { - require_once __DIR__ . "/src/wordpress-core-html-api/html5-named-character-references.php"; +if ( ! isset( $html5_named_character_references ) ) { + require_once __DIR__ . '/src/wordpress-core-html-api/html5-named-character-references.php'; } require_once __DIR__ . '/src/block-markup/WP_Block_Markup_Processor.php'; @@ -56,6 +56,8 @@ require_once __DIR__ . '/src/import/WP_Import_Session.php'; require_once __DIR__ . '/src/import/WP_Stream_Importer.php'; require_once __DIR__ . '/src/import/WP_Markdown_Importer.php'; +require_once __DIR__ . '/src/import/WP_Entity_Iterator_Chain.php'; +require_once __DIR__ . '/src/import/WP_Retry_Frontloading_Iterator.php'; require_once __DIR__ . '/src/utf8_decoder.php'; @@ -65,50 +67,50 @@ require_once __DIR__ . '/vendor/autoload.php'; // Polyfill WordPress core functions -if (!function_exists('_doing_it_wrong')) { - $GLOBALS['_doing_it_wrong_messages'] = []; - function _doing_it_wrong($method, $message, $version) { +if ( ! function_exists( '_doing_it_wrong' ) ) { + $GLOBALS['_doing_it_wrong_messages'] = array(); + function _doing_it_wrong( $method, $message, $version ) { $GLOBALS['_doing_it_wrong_messages'][] = $message; } } -if(!function_exists('wp_kses_uri_attributes')) { +if ( ! function_exists( 'wp_kses_uri_attributes' ) ) { function wp_kses_uri_attributes() { - return []; + return array(); } } -if (!function_exists('__')) { - function __($input) { +if ( ! function_exists( '__' ) ) { + function __( $input ) { return $input; } } -if (!function_exists('esc_attr')) { - function esc_attr($input) { - return htmlspecialchars($input); +if ( ! function_exists( 'esc_attr' ) ) { + function esc_attr( $input ) { + return htmlspecialchars( $input ); } } -if (!function_exists('esc_html')) { - function esc_html($input) { - return htmlspecialchars($input); +if ( ! function_exists( 'esc_html' ) ) { + function esc_html( $input ) { + return htmlspecialchars( $input ); } } -if (!function_exists('esc_url')) { - function esc_url($url) { - return htmlspecialchars($url); +if ( ! function_exists( 'esc_url' ) ) { + function esc_url( $url ) { + return htmlspecialchars( $url ); } } -if (!function_exists('wp_kses_uri_attributes')) { +if ( ! function_exists( 'wp_kses_uri_attributes' ) ) { function wp_kses_uri_attributes() { return array(); } } -if (!function_exists('mbstring_binary_safe_encoding')) { +if ( ! function_exists( 'mbstring_binary_safe_encoding' ) ) { function mbstring_binary_safe_encoding( $reset = false ) { static $encodings = array(); static $overloaded = null; @@ -140,7 +142,7 @@ function mbstring_binary_safe_encoding( $reset = false ) { } } -if (!function_exists('reset_mbstring_encoding')) { +if ( ! function_exists( 'reset_mbstring_encoding' ) ) { function reset_mbstring_encoding() { mbstring_binary_safe_encoding( true ); } diff --git a/packages/playground/data-liberation/data-liberation-page.php b/packages/playground/data-liberation/data-liberation-page.php new file mode 100644 index 00000000000..d91da6f39db --- /dev/null +++ b/packages/playground/data-liberation/data-liberation-page.php @@ -0,0 +1,331 @@ + +
+

Data Liberation

+ +
+

Current Import

+ : + + + Stage: + + +
+
+
    + +
+
+
+
+

Indexing Entities

+ + + + + + + + + + + + +
EntityImportedTotal
+
+ +
+

Downloading Assets

+ +
+ + + / + Files Downloaded + +

Downloads in progress:

+ + + + + + + + + + +
FileProgress
+
+ +
+ + + + + + + + + + + + +
IDFileRetriesStatus
+
+
+ +
+

Importing Entities

+ + + + + + + + + + + + +
EntityImportedTotal
+
+ +
+

Import finished!

+ + Archive the importing session + +
+ +
+ + Stop importing and archive the session + +
+
+
+
+ +
+

Start a new import session

+
+ + + + + + + + + + + + + + + + + + + + + + +
Import Type +
+
+ +
WXR File + +

Upload a WordPress eXtended RSS (WXR) file

+
WXR URL + +

Enter the URL of a WXR file

+
Markdown ZIP + +

Upload a ZIP file containing markdown files

+
+ +

+ +

+
+
+ +

Previous Import Sessions

+ + + + + + + + + + + + + + +
DateData sourceTime takenEntities importedTotal entitiesStatus
No import sessions found
+
\ No newline at end of file diff --git a/packages/playground/data-liberation/import-screen.js b/packages/playground/data-liberation/import-screen.js index 808a0db6a83..7f10b6ec9ce 100644 --- a/packages/playground/data-liberation/import-screen.js +++ b/packages/playground/data-liberation/import-screen.js @@ -1,14 +1,151 @@ -import { store, getContext } from '@wordpress/interactivity'; -const { state } = store('dataLiberation', { +/** + * @TODO + * - Replace window.prompt() calls with a pretty UI-based dialog or + * an inline form or something. + */ +import { store, getContext, getServerContext } from '@wordpress/interactivity'; + +const apiFetch = window.wp.apiFetch; + +const { state, actions } = store('dataLiberation', { state: { - selectedImportType: 'wxr_file', get isImportTypeSelected() { - return getContext().importType === state.selectedImportType; + return getContext()?.importType === state.selectedImportType; + }, + get frontloadingFailed() { + return getContext()?.item.post_status === 'error'; + }, + get isCurrentImportAtStage() { + return getContext()?.stage.id === state.currentImport.stage; + }, + }, + /** + * We're bombarding the server with HTTP requests both to run the import and to + * refresh the reported progress. Do we need such an aggressive refresh rate? + */ + callbacks: { + /** + * Fetches a fresh interactivity state from the server every second. + */ + async startRefreshingProgress() { + while (true) { + if ( + !state.currentImport.active || + state.currentImport.stage === '#finished' + ) { + await new Promise((resolve) => setTimeout(resolve, 1000)); + continue; + } + + const response = await apiFetch({ + path: '/data-liberation/v1/interactivity-state', + }); + Object.assign(state, response); + + await new Promise((resolve) => setTimeout(resolve, 2000)); + } + }, + /** + * Continuously asks the server to continue the import. + * + * @TODO: Ensure two parallel requests are never processing the same import. + * That would lead to race conditions, undefined states, bad stuff in general. + */ + async startImportLoop() { + let failuresInARow = 0; + const maxFailures = 3; + while (true) { + if ( + !state.currentImport.active || + state.currentImport.stage === '#finished' + ) { + await new Promise((resolve) => setTimeout(resolve, 1000)); + continue; + } + + try { + const response = await fetch( + `${window.location.pathname}?page=data-liberation&continue=true`, + { + credentials: 'same-origin', // Preserves cookies + } + ); + const text = await response.text(); + const parser = new DOMParser(); + const doc = parser.parseFromString(text, 'text/html'); + const importOutput = doc.querySelector('#import-output'); + if (importOutput) { + document.querySelector('#import-output').innerHTML = + importOutput.innerHTML; + } + failuresInARow = 0; + } catch (error) { + failuresInARow++; + // TODO: notify the user about the problem. + if (failuresInARow >= maxFailures) { + throw error; + } + } + await new Promise((resolve) => setTimeout(resolve, 1000)); + } }, }, actions: { setImportType: () => { - state.selectedImportType = getContext().importType; + if (getContext()) { + state.selectedImportType = getContext().importType; + } + }, + + async archiveImport() { + window.location.href = `${window.location.pathname}?page=data-liberation&archive=true`; + }, + + // Existing download management actions + async retryDownload(event) { + const postId = event.target.dataset.postId; + const response = await apiFetch({ + path: '/data-liberation/v1/retry-download', + method: 'POST', + data: { post_id: postId }, + }); + + if (response.success) { + window.location.reload(); + } + }, + + async ignoreDownload(event) { + const postId = event.target.dataset.postId; + const response = await apiFetch({ + path: '/data-liberation/v1/ignore-download', + method: 'POST', + data: { post_id: postId }, + }); + + if (response.success) { + window.location.reload(); + } + }, + + async changeDownloadUrl(event) { + const postId = event.target.dataset.postId; + const newUrl = prompt('Enter the new URL for this asset:'); + + if (!newUrl) return; + + const response = await apiFetch({ + path: '/data-liberation/v1/change-download-url', + method: 'POST', + data: { + post_id: postId, + new_url: newUrl, + }, + }); + + if (response.success) { + window.location.reload(); + } }, }, }); diff --git a/packages/playground/data-liberation/plugin.php b/packages/playground/data-liberation/plugin.php index 3c16f5f2e21..94c2b1b3ba8 100644 --- a/packages/playground/data-liberation/plugin.php +++ b/packages/playground/data-liberation/plugin.php @@ -2,6 +2,14 @@ /** * Plugin Name: Data Liberation * Description: Data parsing and importing primitives. + * + * @TODO + * - Get nonces to work + * - Visually appealing UI with smooth transitions between state updates. Right + * now the UI is jerky. We could have beautiful CSS transitions that would + * make the import process feel much smoother. + * - Delete frontloading placeholders that have been successfully downloaded. + * Still keep track of the number of total and successful downloads. */ require_once __DIR__ . '/bootstrap.php'; @@ -24,632 +32,628 @@ * > echo $html->get_updated_html(); * */ -add_filter('wp_kses_uri_attributes', function() { - return []; -}); - -add_action('init', function() { - if ( defined( 'WP_CLI' ) && WP_CLI ) { - /** - * Import a WXR file. - * - * - * : The WXR file to import. - */ - $command = function ( $args, $assoc_args ) { - $file = $args[0]; - data_liberation_import( $file ); - }; - - // Register the WP-CLI import command. - // Example usage: wp data-liberation /path/to/file.xml - WP_CLI::add_command( 'data-liberation', $command ); - } -}); +add_filter( + 'wp_kses_uri_attributes', + function () { + return array(); + } +); + +add_action( + 'init', + function () { + if ( defined( 'WP_CLI' ) && WP_CLI ) { + /** + * Import a WXR file. + * + * + * : The WXR file to import. + */ + $command = function ( $args, $assoc_args ) { + $file = $args[0]; + data_liberation_import( $file ); + }; + + // Register the WP-CLI import command. + // Example usage: wp data-liberation /path/to/file.xml + WP_CLI::add_command( 'data-liberation', $command ); + } + + register_post_status( + 'error', + array( + 'label' => _x( 'Error', 'post' ), // Label name + 'public' => false, + 'exclude_from_search' => false, + 'show_in_admin_all_list' => false, + 'show_in_admin_status_list' => false, + // translators: %s is the number of errors + 'label_count' => _n_noop( 'Error (%s)', 'Error (%s)' ), + ) + ); + } +); // Register admin menu -add_action('admin_menu', function() { - add_menu_page( - 'Data Liberation', - 'Data Liberation', - 'manage_options', - 'data-liberation', - 'data_liberation_admin_page', - 'dashicons-database-import' - ); -}); - -add_action('admin_enqueue_scripts', 'enqueue_data_liberation_scripts'); +add_action( + 'admin_menu', + function () { + add_menu_page( + 'Data Liberation', + 'Data Liberation', + 'manage_options', + 'data-liberation', + 'data_liberation_admin_page', + 'dashicons-database-import' + ); + } +); + +add_action( 'admin_enqueue_scripts', 'enqueue_data_liberation_scripts' ); function enqueue_data_liberation_scripts() { - wp_register_script_module( - '@data-liberation/import-screen', - plugin_dir_url( __FILE__ ) . 'import-screen.js', - array( '@wordpress/interactivity' ) - ); - wp_enqueue_script_module( - '@data-liberation/import-screen', - plugin_dir_url( __FILE__ ) . 'import-screen.js', - array( '@wordpress/interactivity' ) - ); + wp_register_script_module( + '@data-liberation/import-screen', + plugin_dir_url( __FILE__ ) . 'import-screen.js', + array( '@wordpress/interactivity', '@wordpress/interactivity-router', 'wp-api-fetch' ) + ); + wp_enqueue_script( 'wp-api-fetch' ); + wp_enqueue_script_module( + '@data-liberation/import-screen', + plugin_dir_url( __FILE__ ) . 'import-screen.js', + array( '@wordpress/interactivity', '@wordpress/interactivity-router' ) + ); } function data_liberation_add_minute_schedule( $schedules ) { - // add a 'weekly' schedule to the existing set - $schedules['data_liberation_minute'] = array( - 'interval' => 60, - 'display' => __('Once a Minute') - ); - return $schedules; + // add a 'weekly' schedule to the existing set + $schedules['data_liberation_minute'] = array( + 'interval' => 30, + 'display' => __( 'Twice a Minute' ), + ); + return $schedules; } add_filter( 'cron_schedules', 'data_liberation_add_minute_schedule' ); // Render admin page function data_liberation_admin_page() { - $import_session = WP_Import_Session::get_active(); - if($import_session) { - if(isset($_GET['archive'])) { - $import_session->archive(); - echo ''; - exit; - } elseif(isset($_GET['continue'])) { - echo '

Next import step stdout output:

'; - echo '
';
-            data_liberation_process_import();
-            echo '
'; - } - } - - // Populates the initial global state values. - wp_interactivity_state( 'dataLiberation', array( - 'selectedImportType' => 'wxr_file', - 'isImportTypeSelected' => function() { - // @TODO Figure out why this function is not hiding the form rows - $state = wp_interactivity_state(); - $context = wp_interactivity_get_context(); - return $context['importType'] === $state['selectedImportType']; - }, - )); - ?> - -
-

Data Liberation

- - - get_stage(); - $totals = $import_session->get_total_number_of_entities(); - $imported = $import_session->count_imported_entities(); - ?> -

Current Import

- get_data_source(); ?>: - get_human_readable_file_reference(); ?> -
-
-
    -
  • - is_stage_completed(WP_Stream_Importer::STAGE_INITIAL)): ?> - ☑ - - ☐ - - New Import Created -
  • -
  • - is_stage_completed(WP_Stream_Importer::STAGE_INDEX_ENTITIES)): ?> - ☑ - - ☐ - - Index Entities -
  • -
  • - is_stage_completed(WP_Stream_Importer::STAGE_TOPOLOGICAL_SORT)): ?> - ☑ - - ☐ - - Sort Entities -
  • -
  • - is_stage_completed(WP_Stream_Importer::STAGE_FRONTLOAD_ASSETS)): ?> - ☑ - - ☐ - - Assets Download -
  • -
  • - is_stage_completed(WP_Stream_Importer::STAGE_IMPORT_ENTITIES)): ?> - ☑ - - ☐ - - Content Import -
  • -
-
-
- -

New Import Created

- - -

Indexing Entities

- - -

Sorting Entities

-

Determining optimal import order...

- - -

Downloading Assets

- - get_frontloading_progress(); - if (!empty($frontloading_progress)): ?> - - / Files Downloaded - -

Downloads in progress:

- - - - - - - - - $progress): ?> - - - - - - -
FileProgress
- -

Preparing to download assets...

- - - -

Importing Content

- - - - is_stage_completed(WP_Stream_Importer::STAGE_IMPORT_ENTITIES) - ): ?> - - - - - - - - - - $count): ?> - - - - - - - -
EntityImportedTotal
- - -

- Import finished! -

- - Archive the importing session - - - - Continue importing - - - Stop importing and archive the session - - -
-
- - -
-

Start a new import session

-
- - - - - - - - - - - - - - - - - - - - - - -
Import Type -
-
- -
WXR File - -

Upload a WordPress eXtended RSS (WXR) file

-
WXR URL - -

Enter the URL of a WXR file

-
Markdown ZIP - -

Upload a ZIP file containing markdown files

-
- - -
-
- -

Previous Import Sessions

- - - - - - - - - - - WP_Import_Session::POST_TYPE, - 'post_status' => array('archived'), - 'posts_per_page' => -1, - 'orderby' => 'date', - 'order' => 'DESC', - )); - ?> - - - - - - - ID); ?> - - - - - - - - - -
DateData sourceTime takenEntities importedTotal entitiesStatus
No import sessions found
post_date; ?>get_metadata()['data_source']; ?>get_started_at(), $import_session->is_finished() ? $import_session->get_finished_at() : time()); ?>count_imported_entities()); ?>get_total_number_of_entities()); ?>get_stage(); ?>
-
- + + +
+ archive(); + ?> + + get_stage() + ) { + ?> +

Last importer output (for debugging):

+
+ +
+ function () { + $context = wp_interactivity_get_context(); + return $context['item']->post_status === 'error'; + }, + + 'isCurrentImportAtStage' => function () { + $context = wp_interactivity_get_context(); + $state = wp_interactivity_state( 'dataLiberation' ); + return $context['stage']['id'] === $state['currentImport']['stage']; + }, + 'isImportTypeSelected' => function () { + $context = wp_interactivity_get_context(); + $state = wp_interactivity_state( 'dataLiberation' ); + return $context['importType'] === $state['selectedImportType']; + }, + ) + ) + ); + + ob_start(); + include __DIR__ . '/data-liberation-page.php'; + $html = ob_get_clean(); + echo wp_interactivity_process_directives( $html ); } // Handle form submission -add_action('admin_post_data_liberation_import', function() { - if (!current_user_can('manage_options')) { - wp_die('Unauthorized'); - } - - // @TODO: check nonce - // check_admin_nonce('data_liberation_import'); - $data_source = $_POST['data_source']; - $attachment_id = null; - $file_name = ''; - - switch ($data_source) { - case 'wxr_file': - if (empty($_FILES['wxr_file']['tmp_name'])) { - wp_die('Please select a file to upload'); - } - if (!in_array($_FILES['wxr_file']['type'], ['text/xml', 'application/xml'])) { - wp_die('Invalid file type'); - } - /** - * @TODO: Reconsider storing the file in the media library where everyone - * can access it via a public URL. - */ - $attachment_id = media_handle_upload( - 'wxr_file', - 0, - array(), - array( - 'mimes' => array( - 'xml' => 'text/xml', - 'xml-application' => 'application/xml', - ), - // test_form checks: - // Whether to test that the $_POST['action'] parameter is as expected. - // It seems useless here and it causes cryptic error "Invalid form submission". - // Let's just disable it. - 'test_form' => false, - - // @TODO: Find a way to make this type check work. - 'test_type' => false, - ) - ); - if (is_wp_error($attachment_id)) { - wp_die($attachment_id->get_error_message()); - } - $file_name = $_FILES['wxr_file']['name']; - $import_session = WP_Import_Session::create(array( - 'data_source' => 'wxr_file', - 'attachment_id' => $attachment_id, - 'file_name' => $file_name, - )); - break; - - case 'wxr_url': - if (empty($_POST['wxr_url']) || !filter_var($_POST['wxr_url'], FILTER_VALIDATE_URL)) { - wp_die('Please enter a valid URL'); - } - // Don't download the file, it could be 300GB or so. The - // import callback will stream it as needed. - $import_session = WP_Import_Session::create(array( - 'data_source' => 'wxr_url', - 'source_url' => $_POST['wxr_url'], - )); - break; - - case 'markdown_zip': - if (empty($_FILES['markdown_zip']['tmp_name'])) { - wp_die('Please select a file to upload'); - } - if ($_FILES['markdown_zip']['type'] !== 'application/zip') { - wp_die('Invalid file type'); - } - $attachment_id = media_handle_upload('markdown_zip', 0); - if (is_wp_error($attachment_id)) { - wp_die($attachment_id->get_error_message()); - } - $file_name = $_FILES['markdown_zip']['name']; - $import_session = WP_Import_Session::create(array( - 'data_source' => 'markdown_zip', - 'attachment_id' => $attachment_id, - 'file_name' => $file_name, - )); - break; - - default: - wp_die('Invalid import type'); - } - - if ( false === $import_session ) { - // @TODO: More user friendly error message – maybe redirect back to the import screen and - // show the error there. - wp_die('Failed to create an import session'); - } - - // Schedule the next import step every minute, so 30 seconds more than the - // default PHP max_execution_time. - - /** - * @TODO: The schedule doesn't seem to be actually running. - */ - // if(is_wp_error(wp_schedule_event(time(), 'data_liberation_minute', 'data_liberation_process_import'))) { - // wp_delete_attachment($attachment_id, true); - // // @TODO: More user friendly error message – maybe redirect back to the import screen and - // // show the error there. - // wp_die('Failed to schedule import – the "data_liberation_minute" schedule may not be registered.'); - // } - - wp_redirect(add_query_arg( - 'message', 'import-scheduled', - admin_url('admin.php?page=data-liberation') - )); - exit; -}); +add_action( + 'admin_post_data_liberation_import', + function () { + if ( ! current_user_can( 'manage_options' ) ) { + wp_die( 'Unauthorized' ); + } + + // @TODO: check nonce + // check_admin_nonce('data_liberation_import'); + $data_source = $_POST['data_source']; + $attachment_id = null; + $file_name = ''; + + switch ( $data_source ) { + case 'wxr_file': + if ( empty( $_FILES['wxr_file']['tmp_name'] ) ) { + wp_die( 'Please select a file to upload' ); + } + if ( ! in_array( $_FILES['wxr_file']['type'], array( 'text/xml', 'application/xml' ), true ) ) { + wp_die( 'Invalid file type' ); + } + /** + * @TODO: Reconsider storing the file in the media library where everyone + * can access it via a public URL. + */ + $attachment_id = media_handle_upload( + 'wxr_file', + 0, + array(), + array( + 'mimes' => array( + 'xml' => 'text/xml', + 'xml-application' => 'application/xml', + ), + // test_form checks: + // Whether to test that the $_POST['action'] parameter is as expected. + // It seems useless here and it causes cryptic error "Invalid form submission". + // Let's just disable it. + 'test_form' => false, + + // @TODO: Find a way to make this type check work. + 'test_type' => false, + ) + ); + if ( is_wp_error( $attachment_id ) ) { + wp_die( $attachment_id->get_error_message() ); + } + $file_name = $_FILES['wxr_file']['name']; + $import_session = WP_Import_Session::create( + array( + 'data_source' => 'wxr_file', + 'attachment_id' => $attachment_id, + 'file_name' => $file_name, + ) + ); + break; + + case 'wxr_url': + if ( empty( $_POST['wxr_url'] ) || ! filter_var( $_POST['wxr_url'], FILTER_VALIDATE_URL ) ) { + wp_die( 'Please enter a valid URL' ); + } + // Don't download the file, it could be 300GB or so. The + // import callback will stream it as needed. + $import_session = WP_Import_Session::create( + array( + 'data_source' => 'wxr_url', + 'source_url' => $_POST['wxr_url'], + ) + ); + break; + + case 'markdown_zip': + if ( empty( $_FILES['markdown_zip']['tmp_name'] ) ) { + wp_die( 'Please select a file to upload' ); + } + if ( $_FILES['markdown_zip']['type'] !== 'application/zip' ) { + wp_die( 'Invalid file type' ); + } + $attachment_id = media_handle_upload( 'markdown_zip', 0 ); + if ( is_wp_error( $attachment_id ) ) { + wp_die( $attachment_id->get_error_message() ); + } + $file_name = $_FILES['markdown_zip']['name']; + $import_session = WP_Import_Session::create( + array( + 'data_source' => 'markdown_zip', + 'attachment_id' => $attachment_id, + 'file_name' => $file_name, + ) + ); + break; + + default: + wp_die( 'Invalid import type' ); + } + + if ( false === $import_session ) { + // @TODO: More user friendly error message – maybe redirect back to the import screen and + // show the error there. + wp_die( 'Failed to create an import session' ); + } + + // Schedule the next import step every minute, so 30 seconds more than the + // default PHP max_execution_time. + + /** + * @TODO: The schedule doesn't seem to be actually running. + */ + // if(is_wp_error(wp_schedule_event(time(), 'data_liberation_minute', 'data_liberation_process_import'))) { + // wp_delete_attachment($attachment_id, true); + // // @TODO: More user friendly error message – maybe redirect back to the import screen and + // // show the error there. + // wp_die('Failed to schedule import – the "data_liberation_minute" schedule may not be registered.'); + // } + + wp_redirect( + add_query_arg( + 'message', + 'import-scheduled', + admin_url( 'admin.php?page=data-liberation' ) + ) + ); + exit; + } +); // Process import in the background function data_liberation_process_import() { - $session = WP_Import_Session::get_active(); - if (!$session) { - _doing_it_wrong( - __METHOD__, - 'No active import session', - '1.0.0' - ); - return false; - } - return data_liberation_import_step($session); + $session = WP_Import_Session::get_active(); + if ( ! $session ) { + _doing_it_wrong( + __METHOD__, + 'No active import session', + '1.0.0' + ); + return false; + } + return data_liberation_import_step( $session ); +} +add_action( 'data_liberation_process_import', 'data_liberation_process_import' ); + +function data_liberation_import_step( $session ) { + $metadata = $session->get_metadata(); + $importer = data_liberation_create_importer( $metadata ); + if ( ! $importer ) { + return; + } + + /** + * @TODO: Fix this error we get after a few steps: + * Notice: Function WP_XML_Processor::step_in_element was called incorrectly. A tag was not closed. Please see Debugging in WordPress for more information. (This message was added in version WP_VERSION.) in /wordpress/wp-includes/functions.php on line 6114 + */ + $soft_time_limit_seconds = 5; + $hard_time_limit_seconds = 25; + $start_time = microtime( true ); + $fetched_files = 0; + while ( true ) { + $time_taken = microtime( true ) - $start_time; + if ( $time_taken >= $soft_time_limit_seconds ) { + // If we're frontloading and don't have any files fetched yet, + // we need to give it more time. Otherwise every time we retry, + // we'll start from the beginning and never advance past the + // frontloading stage. + if ( $importer->get_stage() === WP_Stream_Importer::STAGE_FRONTLOAD_ASSETS ) { + if ( $fetched_files > 0 ) { + break; + } + } else { + break; + } + } + if ( $time_taken >= $hard_time_limit_seconds ) { + // No negotiation, we're done. + // @TODO: Make it easily configurable + // @TODO: Bump the number of download attempts for the placeholders, + // set the status to `error` in each interrupted download. + break; + } + + if ( true !== $importer->next_step() ) { + $session->set_reentrancy_cursor( $importer->get_reentrancy_cursor() ); + + $should_advance_to_next_stage = null !== $importer->get_next_stage(); + if ( $should_advance_to_next_stage ) { + if ( WP_Stream_Importer::STAGE_FRONTLOAD_ASSETS === $importer->get_stage() ) { + $resolved_all_failures = $session->count_unfinished_frontloading_placeholders() === 0; + if ( ! $resolved_all_failures ) { + break; + } + } + } + if ( ! $importer->advance_to_next_stage() ) { + break; + } + $session->set_stage( $importer->get_stage() ); + $session->set_reentrancy_cursor( $importer->get_reentrancy_cursor() ); + continue; + } + + switch ( $importer->get_stage() ) { + case WP_Stream_Importer::STAGE_INDEX_ENTITIES: + // Bump the total number of entities to import. + $session->create_frontloading_placeholders( $importer->get_indexed_assets_urls() ); + $session->bump_total_number_of_entities( + $importer->get_indexed_entities_counts() + ); + break; + case WP_Stream_Importer::STAGE_FRONTLOAD_ASSETS: + $session->bump_frontloading_progress( + $importer->get_frontloading_progress(), + $importer->get_frontloading_events() + ); + break; + case WP_Stream_Importer::STAGE_IMPORT_ENTITIES: + $session->bump_imported_entities_counts( + $importer->get_imported_entities_counts() + ); + break; + } + + $session->set_reentrancy_cursor( $importer->get_reentrancy_cursor() ); + } } -add_action('data_liberation_process_import', 'data_liberation_process_import'); - -function data_liberation_import_step($session) { - $metadata = $session->get_metadata(); - $importer = data_liberation_create_importer($metadata); - if(!$importer) { - return; - } - /** - * @TODO: Fix this error we get after a few steps: - * Notice: Function WP_XML_Processor::step_in_element was called incorrectly. A tag was not closed. Please see Debugging in WordPress for more information. (This message was added in version WP_VERSION.) in /wordpress/wp-includes/functions.php on line 6114 - */ - - // At this moment, the importer knows where to resume from but - // it hasn't actually pulled the first entity from the stream yet. - // So let's do that now. - if($importer->next_step()) { - // var_dump("Stage: " . $importer->get_stage()); - switch($importer->get_stage()) { - case WP_Stream_Importer::STAGE_INDEX_ENTITIES: - // Bump the total number of entities to import. - var_dump($importer->get_indexed_entities_counts()); - $session->bump_total_number_of_entities([ - ...$importer->get_indexed_entities_counts(), - 'file' => count($importer->get_indexed_assets_urls()) - ]); - break; - case WP_Stream_Importer::STAGE_FRONTLOAD_ASSETS: - var_dump($importer->get_frontloading_progress()); - var_dump($importer->get_frontloading_events()); - $session->bump_frontloading_progress( - $importer->get_frontloading_progress(), - $importer->get_frontloading_events() - ); - break; - case WP_Stream_Importer::STAGE_IMPORT_ENTITIES: - var_dump($importer->get_imported_entities_counts()); - $session->bump_imported_entities_counts( - $importer->get_imported_entities_counts() - ); - break; - } - } - // Move to the next step before saving the cursor so that the next - // import session resumes from the next step. - if(WP_Stream_Importer::STAGE_FRONTLOAD_ASSETS === $importer->get_stage()) { - // Define constraints for this run of the frontloading stage. - // @TODO: Support these constraints at the importer level, not here. - $min_files_downloaded = 0; - $soft_time_limit = 10; - $start_time = time(); - $files_downloaded = 0; - while(true) { - if(!$importer->next_step()) { - break; - } - $frontloading_events = $importer->get_frontloading_events(); - foreach($frontloading_events as $event) { - if($event->type === WP_Attachment_Downloader_Event::SUCCESS) { - ++$files_downloaded; - } - } - $time_taken = time() - $start_time; - if($time_taken > $soft_time_limit) { - if($files_downloaded >= $min_files_downloaded) { - break; - } - } - } - } else { - $importer->next_step(); - } - if($importer->advance_to_next_stage()) { - $session->set_stage($importer->get_stage()); - } - $cursor = $importer->get_reentrancy_cursor(); - if($cursor) { - $session->set_reentrancy_cursor($cursor); - } + +function data_liberation_create_importer( $import ) { + switch ( $import['data_source'] ) { + case 'wxr_file': + $wxr_path = get_attached_file( $import['attachment_id'] ); + if ( false === $wxr_path ) { + // @TODO: Save the error, report it to the user. + return; + } + $importer = WP_Stream_Importer::create_for_wxr_file( + $wxr_path, + array(), + $import['cursor'] ?? null + ); + break; + + case 'wxr_url': + $importer = WP_Stream_Importer::create_for_wxr_url( + $import['wxr_url'], + array(), + $import['cursor'] ?? null + ); + break; + + case 'markdown_zip': + // @TODO: Don't unzip. Stream data directly from the ZIP file. + $zip_path = get_attached_file( $import['attachment_id'] ); + $temp_dir = sys_get_temp_dir() . '/data-liberation-markdown-' . $import['attachment_id']; + if ( ! file_exists( $temp_dir ) ) { + mkdir( $temp_dir, 0777, true ); + $zip = new ZipArchive(); + if ( $zip->open( $zip_path ) === true ) { + $zip->extractTo( $temp_dir ); + $zip->close(); + } else { + // @TODO: Save the error, report it to the user + return; + } + } + $markdown_root = $temp_dir; + $importer = WP_Markdown_Importer::create_for_markdown_directory( + $markdown_root, + array( + 'default_source_site_url' => 'file://' . $markdown_root, + 'local_markdown_assets_root' => $markdown_root, + 'local_markdown_assets_url_prefix' => '@site/', + ), + $import['cursor'] ?? null + ); + break; + } + // @TODO: Consider moving this to the importer constructor. + $retries_iterator = new WP_Retry_Frontloading_Iterator( $import['post_id'] ); + $importer->set_frontloading_retries_iterator( $retries_iterator ); + return $importer; } -function data_liberation_create_importer($import) { - switch($import['data_source']) { - case 'wxr_file': - $wxr_path = get_attached_file($import['attachment_id']); - if(false === $wxr_path) { - // @TODO: Save the error, report it to the user. - return; - } - return WP_Stream_Importer::create_for_wxr_file( - $wxr_path, - [], - $import['cursor'] ?? null - ); - - case 'wxr_url': - return WP_Stream_Importer::create_for_wxr_url( - $import['wxr_url'], - [], - $import['cursor'] ?? null - ); - - case 'markdown_zip': - // @TODO: Don't unzip. Stream data directly from the ZIP file. - $zip_path = get_attached_file($import['attachment_id']); - $temp_dir = sys_get_temp_dir() . '/data-liberation-markdown-' . $import['attachment_id']; - if (!file_exists($temp_dir)) { - mkdir($temp_dir, 0777, true); - $zip = new ZipArchive(); - if ($zip->open($zip_path) === TRUE) { - $zip->extractTo($temp_dir); - $zip->close(); - } else { - // @TODO: Save the error, report it to the user - return; - } - } - $markdown_root = $temp_dir; - return WP_Markdown_Importer::create_for_markdown_directory( - $markdown_root, - [ - 'source_site_url' => 'file://' . $markdown_root, - 'local_markdown_assets_root' => $markdown_root, - 'local_markdown_assets_url_prefix' => '@site/', - ], - $import['cursor'] ?? null - ); - } +function data_liberation_get_interactivity_state() { + // Populates the initial global state values. + $import_history = array_map( + function ( $post ) { + $import_session = new WP_Import_Session( $post->ID ); + return array( + 'date' => $post->post_date, + 'dataSource' => $import_session->get_data_source(), + 'timeTaken' => human_time_diff( $import_session->get_started_at(), $import_session->is_finished() ? $import_session->get_finished_at() : time() ), + 'entitiesImported' => array_sum( array_column( $import_session->count_imported_entities(), 'imported' ) ), + 'totalEntities' => array_sum( array_column( $import_session->count_imported_entities(), 'total' ) ), + 'status' => $import_session->get_stage(), + ); + }, + get_posts( + array( + 'post_type' => WP_Import_Session::POST_TYPE, + 'post_status' => array( 'archived' ), + 'posts_per_page' => -1, + 'orderby' => 'date', + 'order' => 'DESC', + ) + ) + ); + + $import_session = WP_Import_Session::get_active(); + + $stages = array(); + foreach ( WP_Stream_Importer::STAGES_IN_ORDER as $stage ) { + $stages[] = array( + 'id' => $stage, + 'label' => ucfirst( str_replace( '_', ' ', $stage ) ), + 'completed' => $import_session ? $import_session->is_stage_completed( $stage ) : false, + ); + } + + $frontloading_progress = array_map( + function ( $progress, $url ) { + $progress['url'] = $url; + return $progress; + }, + $import_session ? $import_session->get_frontloading_progress() : array(), + array_keys( $import_session ? $import_session->get_frontloading_progress() : array() ) + ); + $frontloading_placeholders = $import_session ? $import_session->get_frontloading_placeholders() : array(); + return array( + // Current import state: + 'currentImport' => $import_session + ? array( + 'active' => true, + 'stage' => $import_session->get_stage(), + 'dataSource' => $import_session->get_data_source(), + 'fileReference' => $import_session->get_human_readable_file_reference(), + 'entityCounts' => $import_session->count_imported_entities(), + 'frontloadingProgress' => $frontloading_progress, + 'hasFrontloadingProgress' => count( $frontloading_progress ) > 0, + 'frontloadingPlaceholders' => $frontloading_placeholders, + 'hasFrontloadingPlaceholders' => count( $frontloading_placeholders ) > 0, + ) + /** + * We need an empty default state that still has all the right data types + * to avoid "undefined index" errors in PHP and ".map is not a function" + * errors in JS. + */ + : array( + 'active' => false, + 'stage' => null, + 'dataSource' => null, + 'fileReference' => null, + 'entityCounts' => array(), + 'frontloadingProgress' => array(), + 'hasFrontloadingProgress' => false, + 'frontloadingPlaceholders' => array(), + 'hasFrontloadingPlaceholders' => false, + ), + + 'stages' => $stages, + + // Import form state: + 'selectedImportType' => 'wxr_file', + + // Past imports table: + 'importHistory' => array( + 'entities' => $import_history, + 'numEntities' => count( $import_history ), + 'page' => 1, + ), + ); } + +add_action( + 'rest_api_init', + function () { + register_rest_route( + 'data-liberation/v1', + '/retry-download', + array( + 'methods' => 'POST', + 'callback' => function ( $request ) { + $post_id = intval( $request->get_param( 'post_id' ) ); + $retry_limit = get_post_meta( $post_id, 'retry_limit', true ); + if ( ! $retry_limit ) { + $retry_limit = 3; + } + update_post_meta( $post_id, 'retry_limit', $retry_limit + 1 ); + + return new WP_REST_Response( array( 'success' => true ), 200 ); + }, + 'permission_callback' => function () { + return current_user_can( 'manage_options' ); + }, + ) + ); + + register_rest_route( + 'data-liberation/v1', + '/ignore-download', + array( + 'methods' => 'POST', + 'callback' => function ( $request ) { + $post_id = intval( $request->get_param( 'post_id' ) ); + wp_update_post( + array( + 'ID' => $post_id, + 'post_status' => WP_Import_Session::FRONTLOAD_STATUS_IGNORED, + ) + ); + + return new WP_REST_Response( array( 'success' => true ), 200 ); + }, + 'permission_callback' => function () { + return current_user_can( 'manage_options' ); + }, + ) + ); + + register_rest_route( + 'data-liberation/v1', + '/change-download-url', + array( + 'methods' => 'POST', + 'callback' => function ( $request ) { + $post_id = intval( $request->get_param( 'post_id' ) ); + $new_url = esc_url_raw( $request->get_param( 'new_url' ) ); + + update_post_meta( $post_id, 'current_url', $new_url ); + update_post_meta( $post_id, 'attempts', 0 ); + update_post_meta( $post_id, 'retry_limit', 3 ); + + return new WP_REST_Response( array( 'success' => true ), 200 ); + }, + 'permission_callback' => function () { + return current_user_can( 'manage_options' ); + }, + ) + ); + + register_rest_route( + 'data-liberation/v1', + '/interactivity-state', + array( + 'methods' => 'GET', + 'callback' => function () { + return new WP_REST_Response( data_liberation_get_interactivity_state(), 200 ); + }, + 'permission_callback' => function () { + return current_user_can( 'manage_options' ); + }, + ) + ); + } +); diff --git a/packages/playground/data-liberation/project.json b/packages/playground/data-liberation/project.json index 44c3e7f5a59..8b8bc80e1b4 100644 --- a/packages/playground/data-liberation/project.json +++ b/packages/playground/data-liberation/project.json @@ -17,7 +17,7 @@ "options": { "cwd": "packages/playground/data-liberation", "commands": [ - "./vendor/bin/phpcs --standard=./phpcs.xml -s ./src" + "./vendor/bin/phpcs --standard=./phpcs.xml -s ./src ./*.php" ], "parallel": false } @@ -27,7 +27,7 @@ "options": { "cwd": "packages/playground/data-liberation", "commands": [ - "./vendor/bin/phpcbf --standard=./phpcs.xml ./src" + "./vendor/bin/phpcbf --standard=./phpcs.xml ./src ./*.php" ], "parallel": false } diff --git a/packages/playground/data-liberation/src/functions.php b/packages/playground/data-liberation/src/functions.php index de37da5bac2..05baf08aa0e 100644 --- a/packages/playground/data-liberation/src/functions.php +++ b/packages/playground/data-liberation/src/functions.php @@ -225,3 +225,12 @@ function data_liberation_import( $path ): bool { return true; } + +function get_all_post_meta_flat( $post_id ) { + return array_map( + function ( $value ) { + return $value[0]; + }, + get_post_meta( $post_id ) + ); +} diff --git a/packages/playground/data-liberation/src/import/WP_Attachment_Downloader.php b/packages/playground/data-liberation/src/import/WP_Attachment_Downloader.php index 261b27c58b4..7dad1f328fe 100644 --- a/packages/playground/data-liberation/src/import/WP_Attachment_Downloader.php +++ b/packages/playground/data-liberation/src/import/WP_Attachment_Downloader.php @@ -43,7 +43,7 @@ public function enqueue_if_not_exists( $url, $output_relative_path ) { if ( file_exists( $output_relative_path ) ) { $this->pending_events[] = new WP_Attachment_Downloader_Event( $this->enqueued_url, - WP_Attachment_Downloader_Event::SUCCESS + WP_Attachment_Downloader_Event::ALREADY_EXISTS ); return true; } @@ -70,10 +70,16 @@ public function enqueue_if_not_exists( $url, $output_relative_path ) { // @TODO: think through the chmod of the created file. $success = copy( $local_path, $output_relative_path ); - $this->pending_events[] = new WP_Attachment_Downloader_Event( - $this->enqueued_url, - $success ? WP_Attachment_Downloader_Event::SUCCESS : WP_Attachment_Downloader_Event::FAILURE - ); + $this->pending_events[] = $success + ? new WP_Attachment_Downloader_Event( + $this->enqueued_url, + WP_Attachment_Downloader_Event::SUCCESS + ) + : new WP_Attachment_Downloader_Event( + $this->enqueued_url, + WP_Attachment_Downloader_Event::FAILURE, + 'copy_failed' + ); return true; case 'http': case 'https': @@ -147,46 +153,64 @@ public function poll() { $this->progress[ $original_url ]['received'] += strlen( $chunk ); break; case Client::EVENT_FAILED: - if ( isset( $this->fps[ $original_request_id ] ) ) { - fclose( $this->fps[ $original_request_id ] ); - } - if ( isset( $this->output_paths[ $original_request_id ] ) ) { - $partial_file = $this->output_paths[ $original_request_id ] . '.partial'; - if ( file_exists( $partial_file ) ) { - unlink( $partial_file ); - } - } - $this->pending_events[] = new WP_Attachment_Downloader_Event( - $original_url, - WP_Attachment_Downloader_Event::FAILURE - ); - unset( $this->progress[ $original_url ] ); - unset( $this->output_paths[ $original_request_id ] ); + $this->on_failure( $original_url, $original_request_id, $request->error ); break; case Client::EVENT_FINISHED: if ( ! $request->is_redirected() ) { - // Only clean up if this was the last request in the chain. - if ( isset( $this->fps[ $original_request_id ] ) ) { - fclose( $this->fps[ $original_request_id ] ); - } - if ( isset( $this->output_paths[ $original_request_id ] ) ) { - if ( false === rename( - $this->output_paths[ $original_request_id ] . '.partial', - $this->output_paths[ $original_request_id ] - ) ) { - // @TODO: Log an error. - } - } - $this->pending_events[] = new WP_Attachment_Downloader_Event( - $original_url, - WP_Attachment_Downloader_Event::SUCCESS + // Only process if this was the last request in the chain. + $is_success = ( + $request->response->status_code >= 200 && + $request->response->status_code <= 299 ); - unset( $this->progress[ $original_url ] ); - unset( $this->output_paths[ $original_request_id ] ); + if ( $is_success ) { + $this->on_success( $original_url, $original_request_id ); + } else { + $this->on_failure( $original_url, $original_request_id, 'http_error_' . $request->response->status_code ); + } } break; } return true; } + + private function on_failure( $original_url, $original_request_id, $error = null ) { + if ( isset( $this->fps[ $original_request_id ] ) ) { + fclose( $this->fps[ $original_request_id ] ); + } + if ( isset( $this->output_paths[ $original_request_id ] ) ) { + $partial_file = $this->output_paths[ $original_request_id ] . '.partial'; + if ( file_exists( $partial_file ) ) { + unlink( $partial_file ); + } + } + $this->pending_events[] = new WP_Attachment_Downloader_Event( + $original_url, + WP_Attachment_Downloader_Event::FAILURE, + $error + ); + unset( $this->progress[ $original_url ] ); + unset( $this->output_paths[ $original_request_id ] ); + } + + private function on_success( $original_url, $original_request_id ) { + // Only clean up if this was the last request in the chain. + if ( isset( $this->fps[ $original_request_id ] ) ) { + fclose( $this->fps[ $original_request_id ] ); + } + if ( isset( $this->output_paths[ $original_request_id ] ) ) { + if ( false === rename( + $this->output_paths[ $original_request_id ] . '.partial', + $this->output_paths[ $original_request_id ] + ) ) { + // @TODO: Log an error. + } + } + $this->pending_events[] = new WP_Attachment_Downloader_Event( + $original_url, + WP_Attachment_Downloader_Event::SUCCESS + ); + unset( $this->progress[ $original_url ] ); + unset( $this->output_paths[ $original_request_id ] ); + } } diff --git a/packages/playground/data-liberation/src/import/WP_Attachment_Downloader_Event.php b/packages/playground/data-liberation/src/import/WP_Attachment_Downloader_Event.php index b759ebb8fb5..0502107782e 100644 --- a/packages/playground/data-liberation/src/import/WP_Attachment_Downloader_Event.php +++ b/packages/playground/data-liberation/src/import/WP_Attachment_Downloader_Event.php @@ -2,14 +2,17 @@ class WP_Attachment_Downloader_Event { - const SUCCESS = '#success'; - const FAILURE = '#failure'; + const SUCCESS = '#success'; + const FAILURE = '#failure'; + const ALREADY_EXISTS = '#already_exists'; public $type; public $resource_id; + public $error; - public function __construct( $resource_id, $type ) { + public function __construct( $resource_id, $type, $error = null ) { $this->resource_id = $resource_id; $this->type = $type; + $this->error = $error; } } diff --git a/packages/playground/data-liberation/src/import/WP_Entity_Iterator_Chain.php b/packages/playground/data-liberation/src/import/WP_Entity_Iterator_Chain.php new file mode 100644 index 00000000000..13128c94f82 --- /dev/null +++ b/packages/playground/data-liberation/src/import/WP_Entity_Iterator_Chain.php @@ -0,0 +1,65 @@ +assets_attempts_iterator = $iterator; + } + + public function set_entities_iterator( Iterator $iterator ) { + $this->entities_iterator = $iterator; + } + + public function current(): mixed { + $iterator = $this->get_iterator(); + return $iterator ? $iterator->current() : null; + } + + public function key(): mixed { + $iterator = $this->get_iterator(); + return $iterator ? $iterator->key() : null; + } + + public function next(): void { + $iterator = $this->get_iterator(); + if ( ! $iterator ) { + return; + } + + $iterator->next(); + } + + public function rewind(): void { + $iterator = $this->get_iterator(); + if ( ! $iterator ) { + return; + } + $iterator->rewind(); + } + + public function valid(): bool { + $iterator = $this->get_iterator(); + return $iterator && $iterator->valid(); + } + + private function get_iterator(): ?Iterator { + if ( $this->assets_attempts_iterator && $this->assets_attempts_iterator->valid() ) { + return $this->assets_attempts_iterator; + } elseif ( $this->entities_iterator && $this->entities_iterator->valid() ) { + return $this->entities_iterator; + } + return null; + } + + public function get_reentrancy_cursor(): mixed { + if ( ! $this->entities_iterator ) { + return null; + } + if ( ! method_exists( $this->entities_iterator, 'get_reentrancy_cursor' ) ) { + return null; + } + return $this->entities_iterator->get_reentrancy_cursor(); + } +} diff --git a/packages/playground/data-liberation/src/import/WP_Import_Session.php b/packages/playground/data-liberation/src/import/WP_Import_Session.php index 13735f58a10..2f56040563f 100644 --- a/packages/playground/data-liberation/src/import/WP_Import_Session.php +++ b/packages/playground/data-liberation/src/import/WP_Import_Session.php @@ -8,11 +8,12 @@ */ class WP_Import_Session { const POST_TYPE = 'import_session'; + /** * @TODO: Make it extendable * @TODO: Reuse the same entities list as WP_Stream_Importer */ - const PROGRESS_ENTITIES = array( + const PROGRESS_ENTITIES = array( 'site_option', 'user', 'category', @@ -22,8 +23,11 @@ class WP_Import_Session { 'post_meta', 'comment', 'comment_meta', - 'file', ); + const FRONTLOAD_STATUS_AWAITING_DOWNLOAD = 'awaiting_download'; + const FRONTLOAD_STATUS_IGNORED = 'ignored'; + const FRONTLOAD_STATUS_ERROR = 'error'; + const FRONTLOAD_STATUS_SUCCEEDED = 'succeeded'; private $post_id; private $cached_stage; @@ -175,6 +179,7 @@ public function get_id() { public function get_metadata() { $cursor = $this->get_reentrancy_cursor(); return array( + 'post_id' => $this->post_id, 'cursor' => $cursor ? $cursor : null, 'data_source' => get_post_meta( $this->post_id, 'data_source', true ), 'source_url' => get_post_meta( $this->post_id, 'source_url', true ), @@ -214,7 +219,11 @@ public function archive() { public function count_imported_entities() { $progress = array(); foreach ( self::PROGRESS_ENTITIES as $entity ) { - $progress[ $entity ] = (int) get_post_meta( $this->post_id, 'imported_' . $entity, true ); + $progress[] = array( + 'label' => $entity, + 'imported' => (int) get_post_meta( $this->post_id, 'imported_' . $entity, true ), + 'total' => (int) get_post_meta( $this->post_id, 'total_' . $entity, true ), + ); } return $progress; } @@ -267,13 +276,154 @@ public function bump_imported_entities_counts( $newly_imported_entities ) { } } + public function count_unfinished_frontloading_placeholders() { + global $wpdb; + return (int) $wpdb->get_var( + $wpdb->prepare( + "SELECT COUNT(*) FROM $wpdb->posts + WHERE post_type = 'frontloading_placeholder' + AND post_parent = %d + AND post_status != %s + AND post_status != %s", + $this->post_id, + self::FRONTLOAD_STATUS_SUCCEEDED, + self::FRONTLOAD_STATUS_IGNORED + ) + ); + } + + public function get_frontloading_placeholders( $options = array() ) { + $query = new WP_Query( + array( + 'post_type' => 'frontloading_placeholder', + 'post_status' => 'any', + 'post_parent' => $this->post_id, + 'posts_per_page' => $options['per_page'] ?? 25, + 'paged' => $options['page'] ?? 1, + 'orderby' => array( + 'post_status' => array( + self::FRONTLOAD_STATUS_ERROR => 0, + self::FRONTLOAD_STATUS_AWAITING_DOWNLOAD => 1, + 'any' => 2, + ), + 'ID' => 'ASC', + ), + ) + ); + + if ( ! $query->have_posts() ) { + return array(); + } + + $posts = $query->posts; + $ids = array_map( + function ( $post ) { + return $post->ID; + }, + $posts + ); + update_meta_cache( 'post', $ids ); + foreach ( $posts as $post ) { + $post->meta = get_all_post_meta_flat( $post->ID ); + } + + return $posts; + } + public function get_total_number_of_entities() { $totals = array(); foreach ( static::PROGRESS_ENTITIES as $field ) { $totals[ $field ] = (int) get_post_meta( $this->post_id, 'total_' . $field, true ); } + $totals['download'] = $this->get_total_number_of_assets(); return $totals; } + + public function get_total_number_of_assets() { + global $wpdb; + return (int) $wpdb->get_var( + $wpdb->prepare( + "SELECT COUNT(*) FROM $wpdb->posts + WHERE post_type = 'frontloading_placeholder' + AND post_parent = %d", + $this->post_id + ) + ); + } + + public function get_frontloading_placeholder( $url ) { + global $wpdb; + $id = $wpdb->get_var( + $wpdb->prepare( + "SELECT p.ID FROM $wpdb->posts p + INNER JOIN $wpdb->postmeta pm ON p.ID = pm.post_id + WHERE p.post_type = 'frontloading_placeholder' + AND p.post_parent = %d + AND pm.meta_key = 'current_url' + AND pm.meta_value = %s + LIMIT 1", + $this->post_id, + $url + ) + ); + return get_post( $id ); + } + + /** + * Creates placeholder attachments for the assets to be downloaded in the + * frontloading stage. + */ + public function create_frontloading_placeholders( $urls ) { + global $wpdb; + + foreach ( $urls as $url => $_ ) { + /** + * Check if placeholder with this URL already exists + * There's a race condition here – another insert may happen + * between the check and the insert. + * @TODO: Explore solutions. A custom table with a UNIQUE constraint + * may or may not be an option, depending on the performance impact + * on 100GB+ VIP databases. + */ + $exists = $wpdb->get_var( + $wpdb->prepare( + "SELECT ID FROM $wpdb->posts + WHERE post_type = 'frontloading_placeholder' + AND post_parent = %d + AND guid = %s + LIMIT 1", + $this->post_id, + $url + ) + ); + + if ( $exists ) { + continue; + } + + $post_data = array( + 'post_type' => 'frontloading_placeholder', + 'post_parent' => $this->post_id, + 'post_title' => basename( $url ), + 'post_status' => self::FRONTLOAD_STATUS_AWAITING_DOWNLOAD, + 'guid' => $url, + 'meta_input' => array( + 'original_url' => $url, + 'current_url' => $url, + 'attempts' => 0, + 'last_error' => null, + 'target_path' => '', + ), + ); + $insertion_result = wp_insert_post( $post_data ); + if ( is_wp_error( $insertion_result ) ) { + // @TODO: How to handle an insertion failure? + throw new Exception( 'Failed to insert frontloading placeholder' ); + return false; + } + } + } + /** * Sets the total number of entities to import for each type. * @@ -314,26 +464,54 @@ public function bump_total_number_of_entities( $newly_indexed_entities ) { public function bump_frontloading_progress( $frontloading_progress, $events = array() ) { update_post_meta( $this->post_id, 'frontloading_progress', $frontloading_progress ); - $successes = 0; foreach ( $events as $event ) { - if ( $event->type === WP_Attachment_Downloader_Event::SUCCESS ) { - ++$successes; - } else { - // @TODO: Store error. + $url = $event->resource_id; + $placeholder = $this->get_frontloading_placeholder( $url ); + if ( ! $placeholder ) { + _doing_it_wrong( + __METHOD__, + 'Frontloading placeholder post not found for URL: ' . $url, + '1.0.0' + ); + continue; + } + + update_post_meta( $placeholder->ID, 'last_error', $event->error ); + + $attempts = get_post_meta( $placeholder->ID, 'attempts', true ); + $new_attempts = $attempts; + switch ( $event->type ) { + case WP_Attachment_Downloader_Event::SUCCESS: + $new_status = self::FRONTLOAD_STATUS_SUCCEEDED; + $new_attempts = $attempts + 1; + break; + case WP_Attachment_Downloader_Event::ALREADY_EXISTS: + $new_status = self::FRONTLOAD_STATUS_SUCCEEDED; + break; + case WP_Attachment_Downloader_Event::FAILURE: + $new_status = self::FRONTLOAD_STATUS_ERROR; + $new_attempts = $attempts + 1; + break; + } + + if ( $new_attempts !== $attempts ) { + update_post_meta( $placeholder->ID, 'attempts', $new_attempts ); + } + + if ( $new_status !== $placeholder->post_status ) { + wp_update_post( + array( + 'ID' => $placeholder->ID, + 'post_status' => $new_status, + ) + ); } - } - if ( $successes > 0 ) { - // @TODO: Consider not treating files as a special case. - $this->bump_imported_entities_counts( - array( - 'file' => $successes, - ) - ); } } public function get_frontloading_progress() { - return get_post_meta( $this->post_id, 'frontloading_progress', true ) ?? array(); + $meta = get_post_meta( $this->post_id, 'frontloading_progress', true ); + return $meta ? $meta : array(); } public function is_stage_completed( $stage ) { diff --git a/packages/playground/data-liberation/src/import/WP_Markdown_Importer.php b/packages/playground/data-liberation/src/import/WP_Markdown_Importer.php index e76e6d643f3..1acc69e5434 100644 --- a/packages/playground/data-liberation/src/import/WP_Markdown_Importer.php +++ b/packages/playground/data-liberation/src/import/WP_Markdown_Importer.php @@ -5,7 +5,7 @@ class WP_Markdown_Importer extends WP_Stream_Importer { public static function create_for_markdown_directory( $markdown_directory, $options = array(), $cursor = null ) { return static::create( function ( $cursor = null ) use ( $markdown_directory ) { - return WP_Markdown_Importer::create( $markdown_directory, $cursor ); + return new WP_Markdown_Directory_Tree_Reader( $markdown_directory, $cursor ); }, $options, $cursor diff --git a/packages/playground/data-liberation/src/import/WP_Retry_Frontloading_Iterator.php b/packages/playground/data-liberation/src/import/WP_Retry_Frontloading_Iterator.php new file mode 100644 index 00000000000..d73357b14ca --- /dev/null +++ b/packages/playground/data-liberation/src/import/WP_Retry_Frontloading_Iterator.php @@ -0,0 +1,98 @@ +import_post_id = $import_post_id; + } + + public function current(): mixed { + if ( ! $this->current ) { + return null; + } + return new WP_Imported_Entity( + 'asset_retry', + array( + 'current_url' => $this->current->meta['current_url'], + 'original_url' => $this->current->meta['original_url'], + ) + ); + } + + public function next(): void { + if ( empty( $this->placeholders ) ) { + $this->query_next_page(); + } + $this->current = array_shift( $this->placeholders ); + $this->rewound = false; + } + + public function key(): mixed { + if ( ! $this->current ) { + return null; + } + return $this->current->ID; + } + + public function valid(): bool { + if ( $this->rewound ) { + $this->next(); + } + return null !== $this->current; + } + + public function rewind(): void { + $this->placeholders = array(); + $this->last_id_on_page = null; + $this->rewound = true; + } + + private function query_next_page() { + global $wpdb; + + $where_clauses = array( + $wpdb->prepare( 'post_type = %s', 'frontloading_placeholder' ), + $wpdb->prepare( 'post_parent = %d', $this->import_post_id ), + $wpdb->prepare( 'post_status = %s', WP_Import_Session::FRONTLOAD_STATUS_ERROR ), + "pm.meta_key = 'attempts'", + 'pm.meta_value < 3', + ); + + if ( null !== $this->last_id_on_page ) { + $where_clauses[] = $wpdb->prepare( 'p.ID > %d', $this->last_id_on_page ); + } + + $where = implode( ' AND ', $where_clauses ); + $this->placeholders = $wpdb->get_results( + "SELECT p.* FROM {$wpdb->posts} p + INNER JOIN {$wpdb->postmeta} pm ON p.ID = pm.post_id + WHERE {$where} + ORDER BY p.ID ASC + LIMIT 100" + ); + $last_placeholder = end( $this->placeholders ); + if ( $last_placeholder ) { + $this->last_id_on_page = $last_placeholder->ID; + } + + update_meta_cache( + 'post', + array_map( + function ( $placeholder ) { + return $placeholder->ID; + }, + $this->placeholders + ) + ); + + foreach ( $this->placeholders as $placeholder ) { + $placeholder->meta = get_all_post_meta_flat( $placeholder->ID ); + } + } +} diff --git a/packages/playground/data-liberation/src/import/WP_Stream_Importer.php b/packages/playground/data-liberation/src/import/WP_Stream_Importer.php index 03e45800a76..05b85ff05b0 100644 --- a/packages/playground/data-liberation/src/import/WP_Stream_Importer.php +++ b/packages/playground/data-liberation/src/import/WP_Stream_Importer.php @@ -14,8 +14,10 @@ * * @TODO: * ✅ Re-entrant import - * * Log errors for the user to decide what to do. - * * Log progress information. + * * Log progress and errors for the user to decide what to do. + * * Indexing + * * ✅ Frontloading + * * Entity import * * Error out if `source_site_url` is not set by the time we're processing * the first encountered URL. * * Disable anything remotely related to KSES during the import. KSES @@ -133,7 +135,11 @@ private function initialize_from_cursor( $cursor ) { return false; } $this->stage = $cursor['stage']; + $this->next_stage = $cursor['next_stage']; $this->resume_at_entity = $cursor['resume_at_entity']; + if ( ! empty( $cursor['source_site_url'] ) ) { + $this->source_site_url = $cursor['source_site_url']; + } return true; } @@ -141,7 +147,13 @@ public function get_reentrancy_cursor() { return json_encode( array( 'stage' => $this->stage, + /** + * Store `next_stage` to distinguish between the start and the end of the entity + * stream. `resume_at_entity` may be null in both cases. + */ + 'next_stage' => $this->next_stage, 'resume_at_entity' => $this->resume_at_entity, + 'source_site_url' => $this->source_site_url, ) ); } @@ -172,11 +184,16 @@ private function __construct( ) { $this->entity_iterator_factory = $entity_iterator_factory; $this->options = $options; - if ( isset( $options['source_site_url'] ) ) { - $this->source_site_url = $options['source_site_url']; + if ( isset( $options['default_source_site_url'] ) ) { + $this->source_site_url = $options['default_source_site_url']; } } + private $frontloading_retries_iterator; + public function set_frontloading_retries_iterator( $frontloading_retries_iterator ) { + $this->frontloading_retries_iterator = $frontloading_retries_iterator; + } + /** * The WordPress entity importer instance. * @TODO: Consider inlining the importer code into this class. @@ -186,9 +203,6 @@ private function __construct( private $importer; public function next_step() { - if ( null !== $this->next_stage ) { - return false; - } switch ( $this->stage ) { case self::STAGE_INITIAL: $this->next_stage = self::STAGE_INDEX_ENTITIES; @@ -241,6 +255,10 @@ public function advance_to_next_stage() { private $indexed_assets_urls = array(); private function index_next_entities( $count = 10000 ) { + if ( null !== $this->next_stage ) { + return false; + } + if ( null === $this->entity_iterator ) { $this->entity_iterator = $this->create_entity_iterator(); } @@ -290,22 +308,29 @@ private function index_next_entities( $count = 10000 ) { * every single post in memory for sure, otherwise WordPress would not * be able to render it, but can we hold all the URLs from 10k posts at once? */ - if ( 'post' === $type ) { - $data = $entity->get_data(); - if ( isset( $data['post_type'] ) && $data['post_type'] === 'attachment' ) { - // @TODO: Consider using sha1 hashes to prevent huge URLs from blowing up the memory. - $this->indexed_assets_urls[ $data['attachment_url'] ] = true; - } elseif ( isset( $data['post_content'] ) ) { - $post = $data; - $p = new WP_Block_Markup_Url_Processor( $post['post_content'], $this->source_site_url ); - while ( $p->next_url() ) { - if ( ! $this->url_processor_matched_asset_url( $p ) ) { - continue; - } + $data = $entity->get_data(); + switch ( $type ) { + case 'site_option': + if ( $data['option_name'] === 'home' ) { + $this->source_site_url = $data['option_value']; + } + break; + case 'post': + if ( isset( $data['post_type'] ) && $data['post_type'] === 'attachment' ) { // @TODO: Consider using sha1 hashes to prevent huge URLs from blowing up the memory. - $this->indexed_assets_urls[ $p->get_raw_url() ] = true; + $this->indexed_assets_urls[ $data['attachment_url'] ] = true; + } elseif ( isset( $data['post_content'] ) ) { + $post = $data; + $p = new WP_Block_Markup_Url_Processor( $post['post_content'], $this->source_site_url ); + while ( $p->next_url() ) { + if ( ! $this->url_processor_matched_asset_url( $p ) ) { + continue; + } + // @TODO: Consider using sha1 hashes to prevent huge URLs from blowing up the memory. + $this->indexed_assets_urls[ $p->get_raw_url() ] = true; + } } - } + break; } $this->entity_iterator->next(); @@ -347,8 +372,9 @@ private function frontloading_advance_reentrancy_cursor() { while ( $this->downloader->next_event() ) { $event = $this->downloader->get_event(); switch ( $event->type ) { - case WP_Attachment_Downloader_Event::SUCCESS: case WP_Attachment_Downloader_Event::FAILURE: + case WP_Attachment_Downloader_Event::SUCCESS: + case WP_Attachment_Downloader_Event::ALREADY_EXISTS: $this->frontloading_events[] = $event; foreach ( array_keys( $this->active_downloads ) as $entity_cursor ) { unset( $this->active_downloads[ $entity_cursor ][ $event->resource_id ] ); @@ -378,8 +404,14 @@ private function frontloading_advance_reentrancy_cursor() { */ private function frontload_next_entity() { if ( null === $this->entity_iterator ) { - $this->entity_iterator = $this->create_entity_iterator(); - $this->downloader = new WP_Attachment_Downloader( $this->options['uploads_path'] ); + $this->entity_iterator = new WP_Entity_Iterator_Chain(); + if ( null !== $this->frontloading_retries_iterator ) { + $this->entity_iterator->set_assets_attempts_iterator( $this->frontloading_retries_iterator ); + } + if ( null === $this->next_stage ) { + $this->entity_iterator->set_entities_iterator( $this->create_entity_iterator() ); + } + $this->downloader = new WP_Attachment_Downloader( $this->options['uploads_path'] ); } // Clear the frontloading events from the previous pass. @@ -403,6 +435,7 @@ private function frontload_next_entity() { * * For large files, every time a full megabyte is downloaded above 10MB. */ if ( true === $this->downloader->poll() ) { + $this->frontloading_advance_reentrancy_cursor(); return true; } } @@ -411,7 +444,7 @@ private function frontload_next_entity() { if ( ! $this->entity_iterator->valid() && ! $this->downloader->has_pending_requests() ) { // This is an assertion to make double sure we're emptying the state queue. if ( ! empty( $this->active_downloads ) ) { - _doing_it_wrong( __METHOD__, 'Frontloading queue is not empty.', '1.0' ); + _doing_it_wrong( __METHOD__, '$active_downloads queue was not empty at the end of the frontloading stage.', '1.0' ); } $this->downloader = null; $this->active_downloads = array(); @@ -431,14 +464,17 @@ private function frontload_next_entity() { $data = $entity->get_data(); switch ( $entity->get_type() ) { - case 'site_option': - if ( $data['option_name'] === 'home' ) { - $this->source_site_url = $data['option_value']; - } + case 'asset_retry': + $this->enqueue_attachment_download( + $data['current_url'], + array( + 'original_url' => $data['original_url'], + ) + ); break; case 'post': if ( isset( $data['post_type'] ) && $data['post_type'] === 'attachment' ) { - $this->enqueue_attachment_download( $data['attachment_url'], null ); + $this->enqueue_attachment_download( $data['attachment_url'] ); } elseif ( isset( $data['post_content'] ) ) { $post = $data; $p = new WP_Block_Markup_Url_Processor( $post['post_content'], $this->source_site_url ); @@ -448,18 +484,15 @@ private function frontload_next_entity() { } $this->enqueue_attachment_download( $p->get_raw_url(), - $post['source_path'] ?? $post['slug'] ?? null + array( + 'context_path' => $post['source_path'] ?? $post['slug'] ?? null, + ) ); } } break; } - /** - * @TODO: Update the progress information. - * @TODO: Save the freshly requested URLs to the cursor. - */ - // Move on to the next entity. $this->entity_iterator->next(); @@ -475,6 +508,10 @@ private function frontload_next_entity() { * the API consumer? */ private function import_next_entity() { + if ( null !== $this->next_stage ) { + return false; + } + $this->imported_entities_counts = array(); if ( null === $this->entity_iterator ) { @@ -567,9 +604,11 @@ public function get_imported_entities_counts() { return $this->imported_entities_counts; } - private function enqueue_attachment_download( string $raw_url, $context_path = null ) { + private function enqueue_attachment_download( string $raw_url, $options = array() ) { + $context_path = $options['context_path'] ?? null; + $original_url = $options['original_url'] ?? $raw_url; $url = $this->rewrite_attachment_url( $raw_url, $context_path ); - $asset_filename = $this->new_asset_filename( $raw_url ); + $asset_filename = $this->new_asset_filename( $original_url ); $output_filename = ltrim( $asset_filename, '/' ); $enqueued = $this->downloader->enqueue_if_not_exists( $url, $output_filename ); diff --git a/packages/playground/data-liberation/src/wxr/WP_WXR_Reader.php b/packages/playground/data-liberation/src/wxr/WP_WXR_Reader.php index 939fff254da..25c21ff6088 100644 --- a/packages/playground/data-liberation/src/wxr/WP_WXR_Reader.php +++ b/packages/playground/data-liberation/src/wxr/WP_WXR_Reader.php @@ -925,6 +925,10 @@ public function valid(): bool { } public function rewind(): void { + // Haven't started yet. + if ( null === $this->last_next_result ) { + return; + } _doing_it_wrong( __METHOD__, 'WP_WXR_Reader does not support rewinding.', diff --git a/packages/playground/data-liberation/tests/wxr/frontloading-attachments-post-contents-and-404s.xml b/packages/playground/data-liberation/tests/wxr/frontloading-attachments-post-contents-and-404s.xml new file mode 100644 index 00000000000..a6d73f0e1c8 --- /dev/null +++ b/packages/playground/data-liberation/tests/wxr/frontloading-attachments-post-contents-and-404s.xml @@ -0,0 +1,116 @@ + + + + + +Theme Accessibility Unit Test Data +http://wpthemetestdata.wordpress.com +Just another WordPress website with a purposefully really long description +Tue, 02 Sep 2014 17:16:26 +0000 +en +1.2 +http://wordpress.com/ +http://wpthemetestdata.wordpress.com + + a11yteam + a11yteam@wordpress.org + + + + + + canola2 + http://wpthemetestdata.wordpress.com/2010/09/10/post-format-gallery/canola2/ + Mon, 10 Jan 2011 13:17:54 +0000 + a11yteam + http://wpthemetestdata.files.wordpress.com/2008/06/canola2.jpg + + + + 611 + 2011-01-10 06:17:54 + 2011-01-10 13:17:54 + open + closed + canola2 + inherit + 555 + 0 + attachment + + 0 + https://wpthemetestdata.files.wordpress.com/2008/06/canola2.jpg + + _wp_attachment_image_alt + + + + _wp_attachment_image_alt + + + + _attachment_original_parent_id + + + + + <![CDATA["The Road Not Taken" by Robert Frost]]> + https://playground.internal/path/?p=1 + Wed, 05 Jun 2024 16:04:48 +0000 + + https://playground.internal/path/?p=1 + + +

Two roads diverged in a yellow wood,
And sorry I could not travel both

+ + + +

+ + +

+]]>
+ + 17973 + + + + + + + + + 0 + 0 + + + 0 + + + + + + + + + +
+
+
\ No newline at end of file