diff --git a/.chronus/changes/loaded-package-is-cli-package-2025-8-26-5-27-22.md b/.chronus/changes/loaded-package-is-cli-package-2025-8-26-5-27-22.md
new file mode 100644
index 00000000000..75cb6e53627
--- /dev/null
+++ b/.chronus/changes/loaded-package-is-cli-package-2025-8-26-5-27-22.md
@@ -0,0 +1,8 @@
+---
+# Change versionKind to one of: internal, fix, dependencies, feature, deprecation, breaking
+changeKind: fix
+packages:
+  - "@typespec/compiler"
+---
+
+LSP connection failure after dynamically loading a certain library in the package
diff --git a/packages/compiler/src/server/lib-provider.ts b/packages/compiler/src/server/lib-provider.ts
index 746fa83dc44..29b766d3dcd 100644
--- a/packages/compiler/src/server/lib-provider.ts
+++ b/packages/compiler/src/server/lib-provider.ts
@@ -57,8 +57,11 @@ export class LibraryProvider {
     const data = await pkg.getPackageJsonData();
     // don't add to cache when failing to load package.json which is unexpected
     if (!data) return false;
+    // Only check dependencies and peerDependencies for @typespec/compiler and exclude devDependencies
+    // to further limit the libraries to load and avoid potential issues cause by loading random libraries.
+    // TODO: add more filter condition when needed
     if (
-      (data.devDependencies && data.devDependencies["@typespec/compiler"]) ||
+      (data.peerDependencies && data.peerDependencies["@typespec/compiler"]) ||
       (data.dependencies && data.dependencies["@typespec/compiler"])
     ) {
       const exports = await pkg.getModuleExports();