diff --git a/src/Stache/Indexes/Index.php b/src/Stache/Indexes/Index.php index 7e437258e70..3b98ab6cf9b 100644 --- a/src/Stache/Indexes/Index.php +++ b/src/Stache/Indexes/Index.php @@ -58,6 +58,17 @@ public function push($value) $this->items[] = $value; } + /** + * Bulk-sets the index items without going through getItems(). Used by + * Store::warmValueIndexes() to write pre-accumulated values in one shot. + */ + public function setItems(array $items): static + { + $this->items = $items; + + return $this; + } + public function load() { if ($this->loaded) { diff --git a/src/Stache/Indexes/Terms/Associations.php b/src/Stache/Indexes/Terms/Associations.php index 1ddef3781e3..7dd927050d7 100644 --- a/src/Stache/Indexes/Terms/Associations.php +++ b/src/Stache/Indexes/Terms/Associations.php @@ -2,32 +2,40 @@ namespace Statamic\Stache\Indexes\Terms; +use Statamic\Facades\Stache; use Statamic\Facades\Taxonomy; use Statamic\Stache\Indexes\Index; +use Statamic\Support\Arr; use Statamic\Support\Str; class Associations extends Index { + /** + * Builds the term→entry association map for a taxonomy from the entries + * stores' value indexes rather than by loading every Entry from disk. + * + * During stache:warm those indexes are already cached (Pass 1 runs before + * this index is built in Pass 2). Outside of warming they'll be built on + * demand, which costs the same as the old query-based approach did. + */ public function getItems() { return Taxonomy::findByHandle($handle = $this->store->childKey()) ->collections() ->flatMap(function ($collection) use ($handle) { - return $collection->queryEntries() - ->where($handle, '<>', null) - ->get() - ->flatMap(function ($entry) use ($handle) { - return collect($entry->value($handle)) - ->map(function ($value) use ($entry) { - return [ - 'value' => $value, - 'slug' => Str::slug($value), - 'entry' => $entry->id(), - 'collection' => $entry->collectionHandle(), - 'site' => $entry->locale(), - ]; - }); - })->all(); + $entries = Stache::store('entries')->store($collection->handle()); + $ids = $entries->index('id'); + $sites = $entries->index('site'); + + return $entries->index($handle)->items() + ->reject(fn ($terms) => is_null($terms)) + ->flatMap(fn ($terms, $key) => collect(Arr::wrap($terms))->map(fn ($term) => [ + 'value' => $term, + 'slug' => Str::slug($term), + 'entry' => $ids->get($key), + 'collection' => $collection->handle(), + 'site' => $sites->get($key), + ])); })->all(); } diff --git a/src/Stache/Stache.php b/src/Stache/Stache.php index 1b25b25591c..1c8e1326758 100644 --- a/src/Stache/Stache.php +++ b/src/Stache/Stache.php @@ -137,7 +137,7 @@ public function warm() if ($this->shouldUseParallelWarming($stores)) { $this->warmInParallel($stores); } else { - $stores->each->warm(); + $this->warmSequentially($stores); } $this->stopTimer(); @@ -280,6 +280,25 @@ protected function shouldUseParallelWarming($stores): bool return true; } + /** + * Two-pass warm: Pass 1 caches all per-item value indexes (including entries' + * taxonomy indexes) across every store before Pass 2 runs. This lets + * Terms\Associations read from the cache in Pass 2 instead of loading all + * Entry objects from disk, which was the main source of slow warm times. + * + * Stores that define their own warm() are warmed through it instead, so + * third-party overrides aren't silently bypassed. + */ + protected function warmSequentially($stores) + { + [$overriding, $twoPass] = $stores->partition->overridesWarm(); + + $overriding->each->warm(); + + $twoPass->each->warmValueIndexes(); + $twoPass->each->warmOtherIndexes(); + } + protected function warmInParallel($stores) { try { @@ -297,7 +316,9 @@ protected function warmInParallel($stores) $closures = $chunks->map(function ($chunk) { return function () use ($chunk) { - return $chunk->each->warm()->keys()->all(); + $this->warmSequentially($chunk); + + return $chunk->keys()->all(); }; })->all(); @@ -310,7 +331,7 @@ protected function warmInParallel($stores) Concurrency::driver($driver)->run($closures); } catch (\Exception $e) { Log::warning('Parallel warming failed, falling back to sequential: '.$e->getMessage()); - $stores->each->warm(); + $this->warmSequentially($stores); } } diff --git a/src/Stache/Stores/AggregateStore.php b/src/Stache/Stores/AggregateStore.php index df641e96186..13cc2fb0741 100644 --- a/src/Stache/Stores/AggregateStore.php +++ b/src/Stache/Stores/AggregateStore.php @@ -87,6 +87,16 @@ public function warm() $this->discoverStores()->each->warm(); } + /** + * AggregateStore's own warm() just fans out to child stores, which is exactly what + * warmValueIndexes()/warmOtherIndexes() do too, so it can still take the two-pass + * path. Subclasses that define their own warm() are reported as overriding it. + */ + public function overridesWarm(): bool + { + return (new \ReflectionMethod($this, 'warm'))->getDeclaringClass()->getName() !== self::class; + } + public function resetMemoizedState() { parent::resetMemoizedState(); @@ -94,6 +104,18 @@ public function resetMemoizedState() $this->stores->each->resetMemoizedState(); } + /** @see Store::warmValueIndexes() */ + public function warmValueIndexes() + { + $this->discoverStores()->each->warmValueIndexes(); + } + + /** @see Store::warmOtherIndexes() */ + public function warmOtherIndexes() + { + $this->discoverStores()->each->warmOtherIndexes(); + } + public function paths() { return $this->discoverStores()->flatMap(function ($store) { diff --git a/src/Stache/Stores/Store.php b/src/Stache/Stores/Store.php index b9b067a76a6..6d11c9de48c 100644 --- a/src/Stache/Stores/Store.php +++ b/src/Stache/Stores/Store.php @@ -409,10 +409,64 @@ public function clear() } public function warm() + { + $this->warmValueIndexes(); + $this->warmOtherIndexes(); + } + + /** + * Whether this store provides its own warm() implementation. Such stores are warmed + * by calling warm() directly rather than through the two-pass split, so third-party + * overrides registered via Stache::registerStore() keep being honoured. + */ + public function overridesWarm(): bool + { + return (new \ReflectionMethod($this, 'warm'))->getDeclaringClass()->getName() !== self::class; + } + + /** + * Pass 1 of the 2-pass warm. Loads every file once and accumulates values for + * all per-item value indexes in a single loop, then writes each index to the + * cache in one shot. This ensures entries' taxonomy indexes (e.g. `categories`) + * are cached before Pass 2 runs, so Terms\Associations can build from them. + */ + public function warmValueIndexes() + { + $valueIndexes = $this->resolveIndexes()->filter(fn ($index) => $this->isPerItemValueIndex($index)); + + if ($valueIndexes->isEmpty()) { + return; + } + + $accumulated = $valueIndexes->map(fn () => [])->all(); + + foreach ($this->paths()->keys() as $key) { + if (! $item = $this->getItem($key)) { + continue; + } + + foreach ($valueIndexes as $name => $index) { + $accumulated[$name][$key] = $index->getItemValue($item); + } + } + + $valueIndexes->each(function ($index, $name) use ($accumulated) { + $index->setItems($accumulated[$name])->cache(); + }); + } + + /** + * Pass 2 of the 2-pass warm. Runs after all stores have completed Pass 1, so + * indexes that depend on other indexes (e.g. Terms\Associations) can read + * them from the cache instead of loading Entry objects from disk. + */ + public function warmOtherIndexes() { $this->shouldCacheFileItems = true; - $this->resolveIndexes()->each->update(); + $this->resolveIndexes() + ->reject(fn ($index) => $this->isPerItemValueIndex($index)) + ->each->update(); $this->shouldCacheFileItems = false; $this->fileItems = null; @@ -426,4 +480,15 @@ public function keys() return $this->keys = (new Keys($this))->load(); } + + /** + * Whether an index can be built purely by mapping getItemValue() over each item. + * Subclasses of Value that override getItems() (e.g. Terms\Value, which merges in + * on-the-fly terms) need their own build logic and belong in Pass 2. + */ + private function isPerItemValueIndex(Index $index): bool + { + return $index instanceof Indexes\Value + && (new \ReflectionMethod($index, 'getItems'))->getDeclaringClass()->getName() === Indexes\Value::class; + } } diff --git a/tests/Stache/StacheTest.php b/tests/Stache/StacheTest.php index 1a6306e8930..6063754d50f 100644 --- a/tests/Stache/StacheTest.php +++ b/tests/Stache/StacheTest.php @@ -2,21 +2,30 @@ namespace Tests\Stache; +use Facades\Tests\Factories\EntryFactory; use Illuminate\Filesystem\Filesystem; use Illuminate\Support\Collection; use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\Attributes\Test; use Statamic\Facades\Blink; +use Statamic\Facades\Blueprint; +use Statamic\Facades\Collection as CollectionFacade; +use Statamic\Facades\Stache as StacheFacade; +use Statamic\Facades\Taxonomy; +use Statamic\Facades\Term; use Statamic\Stache\NullLockStore; use Statamic\Stache\Stache; use Statamic\Stache\Stores\ChildStore; use Statamic\Stache\Stores\CollectionsStore; use Statamic\Stache\Stores\EntriesStore; use Symfony\Component\Lock\LockFactory; +use Tests\PreventSavingStacheItemsToDisk; use Tests\TestCase; class StacheTest extends TestCase { + use PreventSavingStacheItemsToDisk; + protected $stache; public function setUp(): void @@ -195,4 +204,79 @@ public static function watcherProvider() ['production', 'config' => null, 'expected' => false], ]; } + + #[Test] + public function warming_builds_term_associations_for_single_item_taxonomy_fields() + { + Taxonomy::make('tags')->save(); + CollectionFacade::make('blog')->taxonomies(['tags'])->save(); + Blueprint::make('blog')->setNamespace('collections.blog')->setContents(['fields' => [ + ['handle' => 'tags', 'field' => ['type' => 'terms', 'taxonomies' => ['tags'], 'max_items' => 1]], + ]])->save(); + Term::make('alfa')->taxonomy('tags')->data(['title' => 'Alfa'])->save(); + EntryFactory::collection('blog')->id('1')->slug('one')->data(['tags' => 'alfa'])->create(); + EntryFactory::collection('blog')->id('2')->slug('two')->data(['tags' => 'bravo'])->create(); + EntryFactory::collection('blog')->id('3')->slug('three')->data([])->create(); + + StacheFacade::clear(); + StacheFacade::warm(); + + $this->assertEquals([ + ['value' => 'alfa', 'slug' => 'alfa', 'entry' => '1', 'collection' => 'blog', 'site' => 'en'], + ['value' => 'bravo', 'slug' => 'bravo', 'entry' => '2', 'collection' => 'blog', 'site' => 'en'], + ], StacheFacade::store('terms')->store('tags')->index('associations')->items()->all()); + } + + #[Test] + public function warming_includes_on_the_fly_terms_in_term_value_indexes() + { + Taxonomy::make('tags')->save(); + CollectionFacade::make('blog')->taxonomies(['tags'])->save(); + Term::make('alfa')->taxonomy('tags')->data(['title' => 'Alfa'])->save(); + EntryFactory::collection('blog')->id('1')->slug('one')->data(['tags' => ['alfa', 'bravo']])->create(); + + StacheFacade::clear(); + StacheFacade::warm(); + + $store = StacheFacade::store('terms')->store('tags'); + + $this->assertEquals([ + ['value' => 'alfa', 'slug' => 'alfa', 'entry' => '1', 'collection' => 'blog', 'site' => 'en'], + ['value' => 'bravo', 'slug' => 'bravo', 'entry' => '1', 'collection' => 'blog', 'site' => 'en'], + ], $store->index('associations')->items()->all()); + + $this->assertEquals(['en::alfa' => 'alfa', 'en::bravo' => 'bravo'], $store->index('slug')->items()->all()); + $this->assertEquals(['en::alfa' => 'en', 'en::bravo' => 'en'], $store->index('site')->items()->all()); + } + + #[Test] + public function warming_calls_warm_on_stores_that_override_it() + { + $store = new class($this->stache, \Mockery::mock(Filesystem::class)) extends CollectionsStore + { + public $warmed = false; + + public function warm() + { + $this->warmed = true; + } + + public function warmValueIndexes() + { + throw new \Exception('Should not be called on a store that overrides warm().'); + } + + public function warmOtherIndexes() + { + throw new \Exception('Should not be called on a store that overrides warm().'); + } + }; + + $this->stache->registerStore($store); + $this->stache->setLockFactory(new LockFactory(new NullLockStore)); + + $this->stache->warm(); + + $this->assertTrue($store->warmed); + } }