Skip to content
Closed
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -43,4 +43,7 @@ logs
# aem-commerce-prerender related
*.env
*.aio.json
.aem-commerce-prerender.json
.aem-commerce-prerender.json

# AI
.cursor/
124 changes: 124 additions & 0 deletions actions/fetch-all-products/accs.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
/*

Copyright 2026 Adobe. All rights reserved.
This file is licensed to you under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License. You may obtain a copy
of the License at http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software distributed under
the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
OF ANY KIND, either express or implied. See the License for the specific language
governing permissions and limitations under the License.

*/

const {
CategoriesQuery,
ProductCountQuery,
ProductsQuery,
} = require("../queries");
const { requestSaaS } = require("../utils");

// Limiting at 10,000 products per category
const MAX_PAGES_FETCHED = 20;

const accsMapper = ({ productView }) => ({
urlKey: productView.urlKey,
sku: productView.sku,
});

async function getAllCategories(context) {
const categories = [];
const categoriesResp = await requestSaaS(
CategoriesQuery,
"getCategories",
{},
context,
);
const items = categoriesResp.data.categories;
for (const { urlPath, level, name } of items) {
const index = parseInt(level);
categories[index] = categories[index] || [];
categories[index].push({ urlPath, name, level });
}
return categories;
}

async function getSkus(categoryPath, context) {
let productsResp = await requestSaaS(
ProductsQuery,
"getProducts",
{ currentPage: 1, categoryPath },
context,
);
const products = [...productsResp.data.productSearch.items.map(accsMapper)];
let maxPage = productsResp.data.productSearch.page_info.total_pages;

if (maxPage > MAX_PAGES_FETCHED) {
console.warn(`Category ${categoryPath} has more than 10000 products.`);
maxPage = MAX_PAGES_FETCHED;
}

for (let currentPage = 2; currentPage <= maxPage; currentPage++) {
productsResp = await requestSaaS(
ProductsQuery,
"getProducts",
{ currentPage, categoryPath },
context,
);
products.push(...productsResp.data.productSearch.items.map(accsMapper));
}

return products;
}

async function getAllSkus(context) {
const productCountResp = await requestSaaS(
ProductCountQuery,
"getProductCount",
{ categoryPath: "" },
context,
);
const productCount =
productCountResp.data.productSearch?.page_info?.total_pages;

if (!productCount) {
throw new Error("Unknown product count.");
}

if (productCount <= 10000) {
// we can get everything from the default category
return getSkus("", context);
}

const products = new Set();
// we have to traverse the category tree
const categories = await getAllCategories(context);

outer: for (const category of categories) {
if (!category) continue;
while (category.length) {
const slice = category.splice(0, 50);
const fetchedProducts = await Promise.all(
slice.map((cat) => getSkus(cat.urlPath, context)),
);
fetchedProducts
.flatMap((skus) => skus)
.forEach((sku) => products.add(sku));
if (products.size >= productCount) {
// break if we got all products already
break outer;
}
}
}

if (products.size !== productCount) {
console.warn(
`Expected ${productCount} products, but got ${products.size}.`,
);
}

return [...products];
}

module.exports = { getAllSkus };
68 changes: 68 additions & 0 deletions actions/fetch-all-products/aco.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
/*

Copyright 2026 Adobe. All rights reserved.
This file is licensed to you under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License. You may obtain a copy
of the License at http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software distributed under
the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
OF ANY KIND, either express or implied. See the License for the specific language
governing permissions and limitations under the License.

*/

const { requestSaaS } = require("../utils");
const { ProductsQuery } = require("../queries");

// Limiting at 10,000 products per category
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think this is not applicable for the ACO implementation.

Suggested change
// Limiting at 10,000 products per category

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It is applicable to ACO as well. It's a Catalog Service limitation on pagination (500 products per page, max 20 pages). I am reworking this logic to account for category paths related to the configured category families so we can get around this, though.

I think I'll just close this PR and open a new one with the reworked functionality.

const MAX_PAGES_FETCHED = 20;
const CONCURRENCY = 5;
const pLimitPromise = import("p-limit").then(({ default: pLimit }) =>
pLimit(CONCURRENCY),
);

const acoMapper = ({ productView }) => ({
urlKey: productView.urlKey,
sku: productView.sku,
categories: (productView.categories || []).map((c) => c.slug),
});

async function getAllSkus(context) {
const categoryPath = ""; // we are fetching all products from the catalog for ACO
const productsResp = await requestSaaS(
ProductsQuery,
"getProducts",
{ currentPage: 1, categoryPath },
context,
);
const products = productsResp.data.productSearch.items.map(acoMapper);
let maxPage = productsResp.data.productSearch.page_info.total_pages;

if (maxPage > MAX_PAGES_FETCHED) {
console.warn(`Catalog has more than 10000 products.`);
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Wouldn't it be better to make the error message more generic. Otherwise, any change on the pageSize or the MAX_PAGES_FETCHED would the error message be misleading.

Suggested change
console.warn(`Catalog has more than 10000 products.`);
console.warn(`Catalog has more products than the maximum supported. Only the first ${MAX_PAGES_FETCHED} pages will be fetched.`);

maxPage = MAX_PAGES_FETCHED;
}

const limit = await pLimitPromise;
const pages = Array.from({ length: maxPage - 1 }, (_, i) => i + 2);
const results = await Promise.all(
pages.map((page) =>
limit(() =>
requestSaaS(
ProductsQuery,
"getProducts",
{ currentPage: page, categoryPath },
context,
),
),
),
);
for (const resp of results) {
products.push(...resp.data.productSearch.items.map(acoMapper));
}

return products;
}

module.exports = { getAllSkus };
147 changes: 39 additions & 108 deletions actions/fetch-all-products/index.js
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
/*

Copyright 2025 Adobe. All rights reserved.
Copyright 2026 Adobe. All rights reserved.
This file is licensed to you under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License. You may obtain a copy
of the License at http://www.apache.org/licenses/LICENSE-2.0
Expand All @@ -12,129 +12,60 @@ governing permissions and limitations under the License.

*/

const { CategoriesQuery, ProductCountQuery, ProductsQuery } = require('../queries');
const { Core, Files } = require('@adobe/aio-sdk')
const { requestSaaS, FILE_PREFIX } = require('../utils');
const { Timings } = require('../lib/benchmark');
const { getRuntimeConfig } = require('../lib/runtimeConfig');
const { handleActionError } = require('../lib/errorHandler');

async function getSkus(categoryPath, context) {
let productsResp = await requestSaaS(ProductsQuery, 'getProducts', { currentPage: 1, categoryPath }, context);
const products = [...productsResp.data.productSearch.items.map(({ productView }) => (
{
urlKey: productView.urlKey,
sku: productView.sku
}
))];
let maxPage = productsResp.data.productSearch.page_info.total_pages;

if (maxPage > 20) {
console.warn(`Category ${categoryPath} has more than 10000 products.`);
maxPage = 20;
}

for (let currentPage = 2; currentPage <= maxPage; currentPage++) {
productsResp = await requestSaaS(ProductsQuery, 'getProducts', { currentPage, categoryPath }, context);
products.push(...productsResp.data.productSearch.items.map(({ productView }) => (
{
urlKey: productView.urlKey,
sku: productView.sku
}
)));
}

return products;
}

async function getAllCategories(context) {
const categories = [];
const categoriesResp = await requestSaaS(CategoriesQuery, 'getCategories', {}, context);
const items = categoriesResp.data.categories;
for (const {urlPath, level, name} of items) {
const index = parseInt(level);
categories[index] = categories[index] || [];
categories[index].push({urlPath, name, level});
}
return categories;
}

async function getAllSkus(context) {
const productCountResp = await requestSaaS(ProductCountQuery, 'getProductCount', { categoryPath: '' }, context);
const productCount = productCountResp.data.productSearch?.page_info?.total_pages;

if (!productCount) {
throw new Error('Unknown product count.');
}

if (productCount <= 10000) {
// we can get everything from the default category
return getSkus('', context);
}

const products = new Set();
// we have to traverse the category tree
const categories = await getAllCategories(context);

outer: for (const category of categories) {
if (!category) continue;
while (category.length) {
const slice = category.splice(0, 50);
const fetchedProducts = await Promise.all(slice.map((category) => getSkus(category.urlPath, context)));
fetchedProducts.flatMap((skus) => skus).forEach((sku) => products.add(sku));
if (products.size >= productCount) {
// break if we got all products already
break outer;
}
}
}

if (products.size !== productCount) {
console.warn(`Expected ${productCount} products, but got ${products.size}.`);
}

return [...products];
}
const { Core, Files } = require("@adobe/aio-sdk");
const { getConfig, getSiteType, SITE_TYPES, FILE_PREFIX } = require("../utils");
const { Timings } = require("../lib/benchmark");
const { getRuntimeConfig } = require("../lib/runtimeConfig");
const { handleActionError } = require("../lib/errorHandler");
const { getAllSkus: getAllSkusAccs } = require("./accs");
const { getAllSkus: getAllSkusAco } = require("./aco");

async function main(params) {
try {
// Resolve runtime config
const cfg = getRuntimeConfig(params);
const logger = Core.Logger('main', { level: cfg.logLevel });
const logger = Core.Logger("main", { level: cfg.logLevel });

const sharedContext = { ...cfg, logger }
const sharedContext = { ...cfg, logger };

const results = await Promise.all(
cfg.locales.map(async (locale) => {
const context = { ...sharedContext };
if (locale) {
context.locale = locale;
}
const timings = new Timings();
const stateFilePrefix = locale || 'default';
const allSkus = await getAllSkus(context);
timings.sample('getAllSkus');
const filesLib = await Files.init(params.libInit || {});
timings.sample('saveFile');
const productsFileName = `${FILE_PREFIX}/${stateFilePrefix}-products.json`;
await filesLib.write(productsFileName, JSON.stringify(allSkus));
return timings.measures;
})
cfg.locales.map(async (locale) => {
const context = { ...sharedContext };
if (locale) {
context.locale = locale;
}
const timings = new Timings();
const stateFilePrefix = locale || "default";

const siteConfig = await getConfig(context);
const siteType = getSiteType(siteConfig);
const allSkus =
siteType === SITE_TYPES.ACO
? await getAllSkusAco(context)
: await getAllSkusAccs(context);

timings.sample("getAllSkus");
const filesLib = await Files.init(params.libInit || {});
timings.sample("saveFile");
const productsFileName = `${FILE_PREFIX}/${stateFilePrefix}-products.json`;
await filesLib.write(productsFileName, JSON.stringify(allSkus));
return timings.measures;
}),
);

return {
statusCode: 200,
body: { status: 'completed', timings: results }
body: { status: "completed", timings: results },
};
} catch (error) {
// Handle errors and determine if job should fail
const logger = Core.Logger('main', { level: 'error' });
return handleActionError(error, {
logger,
actionName: 'Fetch all products'
const logger = Core.Logger("main", { level: "error" });

return handleActionError(error, {
logger,
actionName: "Fetch all products",
});
}
}

exports.main = main
exports.main = main;
2 changes: 1 addition & 1 deletion actions/pdp-renderer/render.js
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ async function generateProductHtml(sku, urlKey, context) {
return {
title: sanitize(option.title, 'inline'),
id: sanitize(option.id, 'no'),
required: sanitize(option.required, 'no'),
required: sanitize(String(option.required), 'no'),
values: option.values
};
});
Expand Down
Loading