Load terms, meta and comments for batches of posts

features/export.feature

@@ -1365,3 +1365,137 @@ Feature: Export content.
       """
       <wp:meta_value><![CDATA[term_metavalue]]></wp:meta_value>
       """
+
+  Scenario: Export attaches terms, meta and comments to the right posts across batches
+    Given a WP install
+    And I run `wp site empty --yes`
+    And I run `wp post generate --count=150`
+    And I run `wp post create --post_title='Last post' --post_status=publish --tags_input='Zoo,apple,beta' --porcelain`
+    And save STDOUT as {LAST_POST_ID}
+    And I run `wp term create category Zoo --slug=zoo-category`
+    And I run `wp post term add {LAST_POST_ID} category zoo-category`
+    And I run `wp post meta add {LAST_POST_ID} my_meta_key my_meta_value`
+    And I run `wp post meta add {LAST_POST_ID} _edit_lock 123:1`
+    And I run `wp comment create --comment_post_ID={LAST_POST_ID} --comment_content='Approved comment' --comment_approved=1 --porcelain`
+    And save STDOUT as {COMMENT_ID}
+    And I run `wp comment meta add {COMMENT_ID} my_comment_meta_key my_comment_meta_value`
+    And I run `wp comment create --comment_post_ID={LAST_POST_ID} --comment_content='Spam comment' --comment_approved=spam`
+    And a wp-content/mu-plugins/skip-meta.php file:
+      """
+      <?php
+      // Only export meta while the post it belongs to is the current post.
+      add_filter(
+        'wxr_export_skip_postmeta',
+        function ( $skip, $meta_key, $meta ) {
+          return $skip || (int) $meta->post_id !== get_the_ID();
+        },
+        10,
+        3
+      );
+      """
+    And a check-export.php file:
+      """
+      <?php
+      $xml   = simplexml_load_file( $args[0] );
+      $found = 0;
+      foreach ( $xml->channel->item as $item ) {
+        $wp    = $item->children( 'http://wordpress.org/export/1.2/' );
+        $tags  = array();
+        $terms = array();
+        foreach ( $item->category as $category ) {
+          if ( 'post_tag' === (string) $category['domain'] ) {
+            $tags[] = (string) $category['nicename'];
+          }
+          $terms[] = (string) $category;
+        }
+        $meta = array();
+        foreach ( $wp->postmeta as $postmeta ) {
+          if ( in_array( (string) $postmeta->meta_key, array( '_pingme', '_encloseme' ), true ) ) {
+            continue;
+          }
+          $meta[] = (string) $postmeta->meta_key . '=' . (string) $postmeta->meta_value;
+        }
+        $comments = array();
+        foreach ( $wp->comment as $comment ) {
+          $comment_meta = array();
+          foreach ( $comment->commentmeta as $commentmeta ) {
+            $comment_meta[] = (string) $commentmeta->meta_key . '=' . (string) $commentmeta->meta_value;
+          }
+          $comments[] = (string) $comment->comment_content . ' [' . implode( ',', $comment_meta ) . ']';
+        }
+        if ( (string) $wp->post_id === '{LAST_POST_ID}' ) {
+          ++$found;
+          // The terms should be in the same order as the database returns them for the post. Compare
+          // their names, as the database can return terms with the same name in any order.
+          $expected = wp_list_pluck( wp_get_object_terms( (int) $wp->post_id, get_object_taxonomies( 'post' ) ), 'name' );
+          echo 'terms in database order: ' . ( $terms === $expected ? 'yes' : 'no' ) . "\n";
+          sort( $tags );
+          echo 'tags: ' . implode( ',', $tags ) . "\n";
+          echo 'meta: ' . implode( ',', $meta ) . "\n";
+          echo 'comments: ' . implode( ',', $comments ) . "\n";
+        } elseif ( $tags || $meta || $comments || array( 'Uncategorized' ) !== $terms ) {
+          echo 'unexpected data on post ' . $wp->post_id . "\n";
+        }
+      }
+      echo "found: $found\n";
+      """
+
+    When I run `wp export --post_type=post`
+    Then save STDOUT 'Writing to file %s' as {EXPORT_FILE}
+
+    When I run `wp eval-file check-export.php {EXPORT_FILE}`
+    Then STDOUT should be:
+      """
+      terms in database order: yes
+      tags: apple,beta,zoo
+      meta: my_meta_key=my_meta_value
+      comments: Approved comment [my_comment_meta_key=my_comment_meta_value]
+      found: 1
+      """
+
+    # Duplicate IDs in --post__in leave gaps in the keys of the list of post IDs.
+    When I run `wp post list --post_type=post --orderby=ID --order=ASC --format=ids`
+    Then save STDOUT as {POST_IDS}
+
+    When I run `wp eval 'echo implode( ",", array_merge( ...array_map( function ( $id ) { return array( $id, $id ); }, explode( " ", "{POST_IDS}" ) ) ) );'`
+    Then save STDOUT as {DUPLICATED_POST_IDS}
+
+    Given a wp-content/mu-plugins/count-batches.php file:
+      """
+      <?php
+      // Count the queries that load the post meta of a batch of posts.
+      $GLOBALS['batch_loads'] = 0;
+      add_filter(
+        'query',
+        function ( $query ) {
+          if ( false !== strpos( $query, 'postmeta WHERE post_id IN (' ) ) {
+            ++$GLOBALS['batch_loads'];
+          }
+          return $query;
+        }
+      );
+      register_shutdown_function(
+        function () {
+          if ( $GLOBALS['batch_loads'] ) {
+            file_put_contents( ABSPATH . 'batch-loads.txt', $GLOBALS['batch_loads'] );
+          }
+        }
+      );
+      """
+
+    When I run `wp export --post__in={DUPLICATED_POST_IDS} --filename_format=duplicated.xml`
+    Then save STDOUT 'Writing to file %s' as {DUPLICATED_EXPORT_FILE}
+    And the batch-loads.txt file should be:
+      """
+      2
+      """
+
+    When I run `wp eval-file check-export.php {DUPLICATED_EXPORT_FILE}`
+    Then STDOUT should be:
+      """
+      terms in database order: yes
+      tags: apple,beta,zoo
+      meta: my_meta_key=my_meta_value
+      comments: Approved comment [my_comment_meta_key=my_comment_meta_value]
+      found: 1
+      """

src/WP_Export_Query.php

@@ -34,6 +34,20 @@ class WP_Export_Query {
 
 	public $missing_parents = false;
 
+	/**
+	 * Terms, meta and comments of the posts in the current batch, keyed by post ID.
+	 *
+	 * @var array<int, array{terms: array<\WP_Term>, meta: array<\stdClass>, comments: array<\stdClass>}>
+	 */
+	private $batch_data = [];
+
+	/**
+	 * Positions of the post IDs in $post_ids, to find the batch a post belongs to.
+	 *
+	 * @var array<int, int>|null
+	 */
+	private $post_id_positions;
+
 	public function __construct( $filters = [] ) {
 		$this->filters = wp_parse_args( $filters, self::$defaults );
 
@@ -156,9 +170,21 @@ public function exportify_post( $post ) {
 		// phpcs:ignore WordPress.NamingConventions.PrefixAllGlobals.NonPrefixedHooknameFound -- Calling native WordPress hook.
 		$post->post_excerpt = apply_filters( 'the_excerpt_export', $post->post_excerpt );
 		$post->is_sticky    = is_sticky( $post->ID ) ? 1 : 0;
-		$post->terms        = self::get_terms_for_post( $post );
-		$post->meta         = self::get_meta_for_post( $post );
-		$post->comments     = $this->get_comments_for_post( $post );
+
+		if ( ! isset( $this->batch_data[ $post->ID ] ) ) {
+			$this->load_batch_data( $post->ID );
+		}
+
+		if ( isset( $this->batch_data[ $post->ID ] ) ) {
+			$post->terms    = $this->batch_data[ $post->ID ]['terms'];
+			$post->meta     = self::filter_meta_for_export( $this->batch_data[ $post->ID ]['meta'] );
+			$post->comments = $this->batch_data[ $post->ID ]['comments'];
+			unset( $this->batch_data[ $post->ID ] );
+		} else {
+			$post->terms    = self::get_terms_for_post( $post );
+			$post->meta     = self::get_meta_for_post( $post );
+			$post->comments = $this->get_comments_for_post( $post );
+		}
 		// phpcs:ignore WordPress.WP.GlobalVariablesOverride.Prohibited -- Revert back to original.
 		$GLOBALS['post'] = $previous_global_post;
 		return $post;
@@ -413,6 +439,138 @@ private function process_orphaned_terms( $terms ) {
 		return $terms;
 	}
 
+	/**
+	 * Load the terms, meta and comments of the batch of posts starting with the given one.
+	 *
+	 * Querying them for a whole batch at once avoids several queries per exported post.
+	 * The data is the same as what get_terms_for_post(), get_meta_for_post() and
+	 * get_comments_for_post() return for each post, except that the meta is not filtered yet.
+	 *
+	 * @param int $post_id ID of the first post of the batch.
+	 */
+	private function load_batch_data( $post_id ) {
+		global $wpdb;
+
+		if ( null === $this->post_id_positions ) {
+			$this->post_id_positions = array_flip( array_map( 'intval', array_values( $this->post_ids ) ) );
+		}
+
+		$this->batch_data = [];
+		if ( ! isset( $this->post_id_positions[ $post_id ] ) ) {
+			return;
+		}
+
+		// Load the same chunk of posts that the posts iterator queries, which returns them in no particular order.
+		$start    = intdiv( $this->post_id_positions[ $post_id ], self::QUERY_CHUNK ) * self::QUERY_CHUNK;
+		$post_ids = array_map( 'intval', array_slice( $this->post_ids, $start, self::QUERY_CHUNK ) );
+		if ( ! $post_ids ) {
+			return;
+		}
+		$in_ids = implode( ',', $post_ids );
+
+		$terms_by_post    = array_fill_keys( $post_ids, [] );
+		$meta_by_post     = array_fill_keys( $post_ids, [] );
+		$comments_by_post = array_fill_keys( $post_ids, [] );
+
+		// Terms, queried per post type as the taxonomies depend on it.
+		// phpcs:ignore WordPress.DB.PreparedSQL.InterpolatedNotPrepared -- integers only
+		$post_types  = $wpdb->get_results( "SELECT ID, post_type FROM $wpdb->posts WHERE ID IN ($in_ids)" );
+		$ids_by_type = [];
+		foreach ( $post_types as $row ) {
+			$ids_by_type[ $row->post_type ][] = (int) $row->ID;
+		}
+		foreach ( $ids_by_type as $post_type => $ids ) {
+			$taxonomies = get_object_taxonomies( $post_type );
+			if ( empty( $taxonomies ) ) {
+				continue;
+			}
+			$terms = wp_get_object_terms( $ids, $taxonomies, [ 'fields' => 'all_with_object_id' ] );
+			if ( is_wp_error( $terms ) ) {
+				continue;
+			}
+			foreach ( $terms as $term ) {
+				if ( ! $term instanceof \WP_Term ) {
+					continue;
+				}
+				// Set by wp_get_object_terms() for 'all_with_object_id'.
+				$object_id = (int) $term->object_id; // @phpstan-ignore property.notFound
+				if ( isset( $terms_by_post[ $object_id ] ) ) {
+					$terms_by_post[ $object_id ][] = $term;
+				}
+			}
+		}
+
+		// The terms are ordered by name in the database's collation. Order terms with the same name,
+		// e.g. a category and a tag, by their term_taxonomy_id like the query for a single post returns them.
+		foreach ( $terms_by_post as &$terms ) {
+			$terms = self::order_terms_with_same_name( $terms );
+		}
+		unset( $terms );
+
+		// phpcs:ignore WordPress.DB.PreparedSQL.InterpolatedNotPrepared -- integers only
+		$meta_from_db = $wpdb->get_results( "SELECT * FROM $wpdb->postmeta WHERE post_id IN ($in_ids) ORDER BY post_id, meta_id" );
+		// The meta is filtered in exportify_post(), while the post it belongs to is the global post.
+		foreach ( $meta_from_db as $meta ) {
+			$meta_by_post[ (int) $meta->post_id ][] = $meta;
+		}
+
+		if ( ! isset( $this->filters['skip_comments'] ) ) {
+			// phpcs:ignore WordPress.DB.PreparedSQL.InterpolatedNotPrepared -- integers only
+			$comments = $wpdb->get_results( "SELECT * FROM $wpdb->comments WHERE comment_post_ID IN ($in_ids) AND comment_approved <> 'spam' ORDER BY comment_post_ID, comment_ID" );
+			if ( ! empty( $comments ) ) {
+				$comment_meta = [];
+				$comment_ids  = implode( ',', array_map( 'intval', wp_list_pluck( $comments, 'comment_ID' ) ) );
+				// phpcs:ignore WordPress.DB.PreparedSQL.InterpolatedNotPrepared -- integers only
+				foreach ( $wpdb->get_results( "SELECT * FROM $wpdb->commentmeta WHERE comment_id IN ($comment_ids) ORDER BY comment_id, meta_id" ) as $meta ) {
+					$comment_meta[ (int) $meta->comment_id ][] = $meta;
+				}
+
+				foreach ( $comments as $comment ) {
+					$comment->meta = isset( $comment_meta[ (int) $comment->comment_ID ] ) ? $comment_meta[ (int) $comment->comment_ID ] : [];
+					$comments_by_post[ (int) $comment->comment_post_ID ][] = $comment;
+				}
+			}
+		}
+
+		foreach ( $post_ids as $id ) {
+			$this->batch_data[ $id ] = [
+				'terms'    => $terms_by_post[ $id ],
+				'meta'     => $meta_by_post[ $id ],
+				'comments' => $comments_by_post[ $id ],
+			];
+		}
+	}
+
+	/**
+	 * Order runs of terms with the same name by their term_taxonomy_id, keeping the order of the runs.
+	 *
+	 * @param array<\WP_Term> $terms Terms ordered by name.
+	 * @return array<\WP_Term>
+	 */
+	private static function order_terms_with_same_name( $terms ) {
+		$ordered = [];
+		$count   = count( $terms );
+		$i       = 0;
+		while ( $i < $count ) {
+			$run = [ $terms[ $i ] ];
+			++$i;
+			while ( $i < $count && $terms[ $i ]->name === $run[0]->name ) {
+				$run[] = $terms[ $i ];
+				++$i;
+			}
+			if ( count( $run ) > 1 ) {
+				usort(
+					$run,
+					static function ( $a, $b ) {
+						return $a->term_taxonomy_id <=> $b->term_taxonomy_id;
+					}
+				);
+			}
+			array_push( $ordered, ...$run );
+		}
+		return $ordered;
+	}
+
 	private static function get_terms_for_post( $post ) {
 		$taxonomies = get_object_taxonomies( $post->post_type );
 		if ( empty( $taxonomies ) ) {
@@ -425,6 +583,20 @@ private static function get_meta_for_post( $post ) {
 		global $wpdb;
 		$meta_for_export = [];
 		$meta_from_db    = $wpdb->get_results( $wpdb->prepare( "SELECT * FROM $wpdb->postmeta WHERE post_id = %d", $post->ID ) );
+		return self::filter_meta_for_export( $meta_from_db );
+	}
+
+	/**
+	 * Remove the meta that should not be exported.
+	 *
+	 * Called while the post the meta belongs to is the global post, as callbacks
+	 * of the `wxr_export_skip_postmeta` filter may rely on it.
+	 *
+	 * @param array<\stdClass> $meta_from_db Meta rows of a post.
+	 * @return array<\stdClass>
+	 */
+	private static function filter_meta_for_export( $meta_from_db ) {
+		$meta_for_export = [];
 		foreach ( $meta_from_db as $meta ) {
 			if ( '_edit_lock' === $meta->meta_key ) {
 				continue;

src/WP_Export_WXR_Formatter.php

@@ -288,7 +288,10 @@ protected function terms( $terms ) {
 
 	protected function comment_meta( $comment ) {
 		global $wpdb;
-		$metas = $wpdb->get_results( $wpdb->prepare( "SELECT * FROM $wpdb->commentmeta WHERE comment_id = %d", $comment->comment_ID ) );
+		// The export query already loads the comment meta, see WP_Export_Query::exportify_post().
+		$metas = isset( $comment->meta ) && is_array( $comment->meta )
+			? $comment->meta
+			: $wpdb->get_results( $wpdb->prepare( "SELECT * FROM $wpdb->commentmeta WHERE comment_id = %d", $comment->comment_ID ) );
 		if ( ! $metas ) {
 			return new Oxymel();
 		}