Merge pull request #449 from sij411/feat/optique-traverse

Support multiple URLs for -t/--traverse in cli command fedify lookup

Author: dahlia

CHANGES.md

@@ -84,8 +84,14 @@ To be released.
         to trash and continue the initialization from new created directory.
      -  Ask again if some options is not specified or invalid.
 
+ -  The `fedify lookup` command now supports multiple URLs with the
+    `-t`/`--traverse` option, allowing users to traverse multiple collections
+    in a single command.  [[#408], [#449] by Jiwon Kwon]
+
 [#397]: https://github.com/fedify-dev/fedify/issues/397
+[#408]: https://github.com/fedify-dev/fedify/issues/408
 [#435]: https://github.com/fedify-dev/fedify/issues/435
+[#449]: https://github.com/fedify-dev/fedify/pull/449
 
 
 Version 1.9.0

packages/cli/src/lookup.ts

@@ -69,7 +69,7 @@ const traverseOption = withDefault(
   object({
     traverse: flag("-t", "--traverse", {
       description:
-        message`Traverse the given collection to fetch all items. If it is turned on, the argument cannot be multiple.`,
+        message`Traverse the given collection(s) to fetch all items.`,
     }),
     suppressErrors: option("-S", "--suppress-errors", {
       description:
@@ -276,15 +276,9 @@ function handleTimeoutError(
 }
 
 export async function runLookup(command: InferValue<typeof lookupCommand>) {
-  // FIXME: Implement -t, --traverse when multiple URLs are provided
   if (command.urls.length < 1) {
     printError(message`At least one URL or actor handle must be provided.`);
     process.exit(1);
-  } else if (command.traverse && command.urls.length > 1) {
-    printError(
-      message`The -t/--traverse option cannot be used with multiple arguments.`,
-    );
-    process.exit(1);
   }
 
   // Enable Debug mode if requested
@@ -389,84 +383,103 @@ export async function runLookup(command: InferValue<typeof lookupCommand>) {
   }...`;
 
   if (command.traverse) {
-    const url = command.urls[0];
-    let collection: APObject | null;
-    try {
-      collection = await lookupObject(url, {
-        documentLoader: authLoader ?? documentLoader,
-        contextLoader,
-        userAgent: command.userAgent,
-      });
-    } catch (error) {
-      if (error instanceof TimeoutError) {
-        handleTimeoutError(spinner, command.timeout, url);
-      } else {
+    let totalItems = 0;
+
+    for (let urlIndex = 0; urlIndex < command.urls.length; urlIndex++) {
+      const url = command.urls[urlIndex];
+
+      if (urlIndex > 0) {
+        spinner.text = `Looking up collection ${
+          urlIndex + 1
+        }/${command.urls.length}...`;
+      }
+
+      let collection: APObject | null;
+      try {
+        collection = await lookupObject(url, {
+          documentLoader: authLoader ?? documentLoader,
+          contextLoader,
+          userAgent: command.userAgent,
+        });
+      } catch (error) {
+        if (error instanceof TimeoutError) {
+          handleTimeoutError(spinner, command.timeout, url);
+        } else {
+          spinner.fail(`Failed to fetch object: ${colors.red(url)}.`);
+          if (authLoader == null) {
+            printError(
+              message`It may be a private object.  Try with -a/--authorized-fetch.`,
+            );
+          }
+        }
+        await server?.close();
+        process.exit(1);
+      }
+      if (collection == null) {
         spinner.fail(`Failed to fetch object: ${colors.red(url)}.`);
         if (authLoader == null) {
           printError(
             message`It may be a private object.  Try with -a/--authorized-fetch.`,
           );
         }
+        await server?.close();
+        process.exit(1);
       }
-      await server?.close();
-      process.exit(1);
-    }
-    if (collection == null) {
-      spinner.fail(`Failed to fetch object: ${colors.red(url)}.`);
-      if (authLoader == null) {
-        printError(
-          message`It may be a private object.  Try with -a/--authorized-fetch.`,
-        );
-      }
-      await server?.close();
-      process.exit(1);
-    }
-    if (!(collection instanceof Collection)) {
-      spinner.fail(
-        `Not a collection: ${colors.red(url)}.  ` +
-          "The -t/--traverse option requires a collection.",
-      );
-      await server?.close();
-      process.exit(1);
-    }
-    spinner.succeed(`Fetched collection: ${colors.green(url)}.`);
-
-    try {
-      let i = 0;
-      for await (
-        const item of traverseCollection(collection, {
-          documentLoader: authLoader ?? documentLoader,
-          contextLoader,
-          suppressError: command.suppressErrors,
-        })
-      ) {
-        if (!command.output && i > 0) print(message`${command.separator}`);
-        await writeObjectToStream(
-          item,
-          command.output,
-          command.format,
-          contextLoader,
+      if (!(collection instanceof Collection)) {
+        spinner.fail(
+          `Not a collection: ${colors.red(url)}.  ` +
+            "The -t/--traverse option requires a collection.",
         );
-        i++;
+        await server?.close();
+        process.exit(1);
       }
-    } catch (error) {
-      logger.error("Failed to complete the traversal: {error}", { error });
-      if (error instanceof TimeoutError) {
-        handleTimeoutError(spinner, command.timeout);
-      } else {
-        spinner.fail("Failed to complete the traversal.");
-        if (authLoader == null) {
-          printError(
-            message`It may be a private object.  Try with -a/--authorized-fetch.`,
+      spinner.succeed(`Fetched collection: ${colors.green(url)}.`);
+
+      try {
+        let collectionItems = 0;
+        for await (
+          const item of traverseCollection(collection, {
+            documentLoader: authLoader ?? documentLoader,
+            contextLoader,
+            suppressError: command.suppressErrors,
+          })
+        ) {
+          if (!command.output && (totalItems > 0 || collectionItems > 0)) {
+            print(message`${command.separator}`);
+          }
+          await writeObjectToStream(
+            item,
+            command.output,
+            command.format,
+            contextLoader,
           );
+          collectionItems++;
+          totalItems++;
+        }
+      } catch (error) {
+        logger.error("Failed to complete the traversal for {url}: {error}", {
+          url,
+          error,
+        });
+        if (error instanceof TimeoutError) {
+          handleTimeoutError(spinner, command.timeout, url);
         } else {
-          printError(
-            message`Use the -S/--suppress-errors option to suppress partial errors.`,
+          spinner.fail(
+            `Failed to complete the traversal for: ${colors.red(url)}.`,
           );
+          if (authLoader == null) {
+            printError(
+              message`It may be a private object.  Try with -a/--authorized-fetch.`,
+            );
+          } else {
+            printError(
+              message`Use the -S/--suppress-errors option to suppress partial errors.`,
+            );
+          }
         }
+        await server?.close();
+        process.exit(1);
       }
-      await server?.close();
-      process.exit(1);
     }
     spinner.succeed("Successfully fetched all items in the collection.");
 
@@ -495,8 +508,7 @@ export async function runLookup(command: InferValue<typeof lookupCommand>) {
   try {
     objects = await Promise.all(promises);
   } catch (_error) {
-    //TODO: implement -a --authorized-fetch
-    // await server?.close();
+    await server?.close();
     process.exit(1);
   }