Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 7 additions & 7 deletions .github/workflows/pullRequests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ jobs:
run: yarn --immutable
working-directory: ${{ github.base_ref }}
- name: Build packages
run: yarn build --rebuild-dependents
run: yarn build
working-directory: ${{ github.base_ref }}
- uses: actions/cache@v5
with:
Expand Down Expand Up @@ -149,7 +149,7 @@ jobs:
run: yarn --immutable
working-directory: ${{ github.base_ref }}
- name: Build packages
run: yarn build --rebuild-dependents
run: yarn build
working-directory: ${{ github.base_ref }}
- name: Build core
run: yarn webiny build core
Expand Down Expand Up @@ -233,7 +233,7 @@ jobs:
run: yarn --immutable
working-directory: ${{ github.base_ref }}
- name: Build packages
run: yarn build --rebuild-dependents
run: yarn build
working-directory: ${{ github.base_ref }}
- name: Sync Dependencies Verification
run: yarn verify-dependencies
Expand Down Expand Up @@ -335,7 +335,7 @@ jobs:
run: yarn --immutable
working-directory: ${{ github.base_ref }}
- name: Build packages
run: yarn build --rebuild-dependents
run: yarn build
working-directory: ${{ github.base_ref }}
- name: Run tests
run: ${{ matrix.testCommand.cmd }}
Expand Down Expand Up @@ -418,7 +418,7 @@ jobs:
run: yarn --immutable
working-directory: ${{ github.base_ref }}
- name: Build packages
run: yarn build --rebuild-dependents
run: yarn build
working-directory: ${{ github.base_ref }}
- name: Run tests
run: ${{ matrix.testCommand.cmd }}
Expand Down Expand Up @@ -511,7 +511,7 @@ jobs:
run: yarn --immutable
working-directory: ${{ github.base_ref }}
- name: Build packages
run: yarn build --rebuild-dependents
run: yarn build
working-directory: ${{ github.base_ref }}
- name: Run tests
run: ${{ matrix.testCommand.cmd }}
Expand Down Expand Up @@ -596,7 +596,7 @@ jobs:
run: yarn --immutable
working-directory: ${{ github.base_ref }}
- name: Build packages
run: yarn build --rebuild-dependents
run: yarn build
working-directory: ${{ github.base_ref }}
- name: Run tests
run: ${{ matrix.testCommand.cmd }}
Expand Down
16 changes: 8 additions & 8 deletions .github/workflows/push.yml
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ jobs:
run: yarn --immutable
working-directory: v6
- name: Build packages
run: yarn build --rebuild-dependents
run: yarn build
working-directory: v6
- uses: actions/cache@v5
with:
Expand Down Expand Up @@ -133,7 +133,7 @@ jobs:
run: yarn --immutable
working-directory: v6
- name: Build packages
run: yarn build --rebuild-dependents
run: yarn build
working-directory: v6
- name: Sync Dependencies Verification
run: yarn verify-dependencies
Expand Down Expand Up @@ -226,7 +226,7 @@ jobs:
run: yarn --immutable
working-directory: v6
- name: Build packages
run: yarn build --rebuild-dependents
run: yarn build
working-directory: v6
- name: Run tests
run: ${{ matrix.testCommand.cmd }}
Expand Down Expand Up @@ -300,7 +300,7 @@ jobs:
run: yarn --immutable
working-directory: v6
- name: Build packages
run: yarn build --rebuild-dependents
run: yarn build
working-directory: v6
- name: Run tests
run: ${{ matrix.testCommand.cmd }}
Expand Down Expand Up @@ -382,7 +382,7 @@ jobs:
run: yarn --immutable
working-directory: v6
- name: Build packages
run: yarn build --rebuild-dependents
run: yarn build
working-directory: v6
- name: Run tests
run: ${{ matrix.testCommand.cmd }}
Expand Down Expand Up @@ -458,7 +458,7 @@ jobs:
run: yarn --immutable
working-directory: v6
- name: Build packages
run: yarn build --rebuild-dependents
run: yarn build
working-directory: v6
- name: Run tests
run: ${{ matrix.testCommand.cmd }}
Expand Down Expand Up @@ -534,7 +534,7 @@ jobs:
run: yarn --immutable
working-directory: v6
- name: Build packages
run: yarn build --rebuild-dependents
run: yarn build
working-directory: v6
- name: Start Verdaccio local server
working-directory: v6
Expand Down Expand Up @@ -695,7 +695,7 @@ jobs:
run: yarn --immutable
working-directory: v6
- name: Build packages
run: yarn build --rebuild-dependents
run: yarn build
working-directory: v6
- name: Start Verdaccio local server
working-directory: v6
Expand Down
3 changes: 1 addition & 2 deletions .github/workflows/wac/pullRequests.wac.ts
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,7 @@ import {
const DIR_WEBINY_JS = "${{ github.base_ref }}";

const installBuildSteps = createInstallBuildSteps({
workingDirectory: DIR_WEBINY_JS,
rebuildDependents: true
workingDirectory: DIR_WEBINY_JS
});
const yarnCacheSteps = createYarnCacheSteps({ workingDirectory: DIR_WEBINY_JS });
const globalBuildCacheSteps = createGlobalBuildCacheSteps({ workingDirectory: DIR_WEBINY_JS });
Expand Down
3 changes: 1 addition & 2 deletions .github/workflows/wac/push.wac.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,7 @@ const DIR_WEBINY_JS = "v6";
const DIR_TEST_PROJECT = "new-webiny-project";

const installBuildSteps = createInstallBuildSteps({
workingDirectory: DIR_WEBINY_JS,
rebuildDependents: true
workingDirectory: DIR_WEBINY_JS
});
const yarnCacheSteps = createYarnCacheSteps({
workingDirectory: DIR_WEBINY_JS
Expand Down
8 changes: 3 additions & 5 deletions .github/workflows/wac/steps/createInstallBuildSteps.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,17 +2,15 @@ import { withCommonParams } from "./withCommonParams.js";

interface CreateInstallBuildStepsParams {
workingDirectory: string;
rebuildDependents?: boolean;
}

export const createInstallBuildSteps = (params: CreateInstallBuildStepsParams) => {
const rebuildDependents = params.rebuildDependents ?? false;
const buildCommand = rebuildDependents ? "yarn build --rebuild-dependents" : "yarn build";

return withCommonParams(
[
{ name: "Install dependencies", run: "yarn --immutable" },
{ name: "Build packages", run: buildCommand }
// The build's dependency-aware cache key rebuilds dependents of any
// changed package on its own — no `--rebuild-dependents` needed.
{ name: "Build packages", run: "yarn build" }
],
{ "working-directory": params.workingDirectory }
);
Expand Down
4 changes: 1 addition & 3 deletions scripts/buildPackages/src/buildPackages.ts
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@ interface BuildOptions {
p?: string | string[];
debug?: boolean;
cache?: boolean;
rebuildDependents?: boolean;
buildOverrides?: string;
safeReplace?: boolean;
}
Expand Down Expand Up @@ -64,8 +63,7 @@ export const buildPackages = async () => {

const { batches, packagesNoCache, allPackages, buildKeys } = await getBatches({
cache: options.cache ?? true,
packagesWhitelist,
rebuildDependents: options.rebuildDependents
packagesWhitelist
});

if (!packagesNoCache.length) {
Expand Down
48 changes: 9 additions & 39 deletions scripts/buildPackages/src/getBatches.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,13 @@ import { getBuildOutputFolder } from "./getBuildOutputFolder";
import { getBuildMeta } from "./getBuildMeta";
import { getPackageCacheFolderPath } from "./getPackageCacheFolderPath";
import { distMatchesCache, recordCacheHash } from "./distContentHash";
import { getEffectiveHashes, getOwnHashes, isDepAwareKeyEnabled } from "./getEffectiveHashes";
import { getEffectiveHashes } from "./getEffectiveHashes";

const { green } = chalk;

interface GetBatchesOptions {
cache?: boolean;
packagesWhitelist?: string[];
rebuildDependents?: boolean;
}

export async function getBatches(options: GetBatchesOptions = {}) {
Expand Down Expand Up @@ -46,13 +45,11 @@ export async function getBatches(options: GetBatchesOptions = {}) {
ignore: ["@webiny/project-utils"]
});

// Build key per package. Default: own-source hash (original behavior).
// Experimental (WEBINY_EXPERIMENTAL_DEP_AWARE_CACHE): a dependency-aware key
// that also changes when any transitive dependency changes, so dependents of
// a changed package are detected as misses without `--rebuild-dependents`.
const buildKeys = isDepAwareKeyEnabled()
? await getEffectiveHashes(allWorkspacePackages)
: await getOwnHashes(allWorkspacePackages);
// Dependency-aware build key per package: changes when the package's own
// source, any transitive workspace dependency, or any resolved third-party
// dependency changes — so dependents of a changed package are detected as
// misses on a plain build.
const buildKeys = await getEffectiveHashes(allWorkspacePackages);

// 1. Determine for which packages we can use the cached built code, and for which we need to execute build.
if (!useCache) {
Expand All @@ -77,36 +74,9 @@ export async function getBatches(options: GetBatchesOptions = {}) {
}
}

// 1.5 When using cache and --rebuild-dependents, also rebuild any package that depends on a changed package.
if (options.rebuildDependents && packagesNoCache.length > 0 && useCache) {
const dependents = workspaceGraph.getDependents();

const tainted = new Set(packagesNoCache.map(p => p.packageJson.name));
const queue = [...tainted];
while (queue.length > 0) {
const name = queue.pop()!;
for (const dependent of dependents.get(name) || []) {
if (!tainted.has(dependent)) {
tainted.add(dependent);
queue.push(dependent);
}
}
}

for (const name of tainted) {
if (packagesNoCache.some(p => p.packageJson.name === name)) continue;
const pkg = workspacesPackages.find(p => p.packageJson.name === name);
if (pkg) {
packagesNoCache.push(pkg);
}
}

for (let i = packagesUseCache.length - 1; i >= 0; i--) {
if (tainted.has(packagesUseCache[i].packageJson.name)) {
packagesUseCache.splice(i, 1);
}
}
}
// Dependents of a changed package no longer need explicit tainting: the
// dependency-aware key folds in dependency keys, so any dependent of a
// changed package is already a cache miss above.

// 2. Let's use cached built code where possible.
if (packagesUseCache.length) {
Expand Down
27 changes: 0 additions & 27 deletions scripts/buildPackages/src/getEffectiveHashes.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,33 +7,6 @@ import { PROJECT_ROOT } from "../../utils/getPackages.js";
import { getPackageSourceHash } from "./getPackageSourceHash";
import type { Package } from "./types";

/**
* Whether the experimental dependency-aware build key is enabled. OFF by
* default: the build key is each package's own-source hash (original behavior),
* and `--rebuild-dependents` remains the mechanism for rebuilding dependents.
*
* Enable with `WEBINY_EXPERIMENTAL_DEP_AWARE_CACHE=true` (or `1`) to make a
* plain `yarn build` rebuild dependents of any changed package automatically.
*/
export function isDepAwareKeyEnabled(): boolean {
const value = process.env.WEBINY_EXPERIMENTAL_DEP_AWARE_CACHE;
return value === "true" || value === "1";
}

/**
* Own-source hash per package (no dependency folding) — the original,
* non-dependency-aware build key. Used when the experimental key is disabled.
*/
export async function getOwnHashes(allPackages: Package[]): Promise<Map<string, string>> {
const map = new Map<string, string>();
await Promise.all(
allPackages.map(async pkg => {
map.set(pkg.name, await getPackageSourceHash(pkg));
})
);
return map;
}

/**
* Parses `yarn.lock` into a map of dependency descriptor → a token that changes
* whenever the *resolved* package changes (its checksum, falling back to
Expand Down
Loading