Add lock to .daml folder for package db (#21180) (#21277)

* Add lock to .daml folder for package db

* Fix import

* Move sdk version check/cache hit into the lock

Author: samuel-williams-da

sdk/compiler/damlc/BUILD.bazel

@@ -185,6 +185,7 @@ da_haskell_library(
         "either",
         "extra",
         "file-embed",
+        "filelock",
         "filepath",
         "ghcide",
         "ghc-lib",

sdk/compiler/damlc/lib/DA/Cli/Damlc.hs

@@ -64,8 +64,7 @@ import DA.Cli.Damlc.Command.MultiIde (runMultiIde)
 import DA.Cli.Damlc.Command.UpgradeCheck (runUpgradeCheck)
 import qualified DA.Daml.Dar.Reader as InspectDar
 import qualified DA.Cli.Damlc.Command.Damldoc as Damldoc
-import DA.Cli.Damlc.Packaging (createProjectPackageDb, mbErr)
-import DA.Cli.Damlc.DependencyDb (installDependencies)
+import DA.Cli.Damlc.Packaging (setupPackageDb, setupPackageDbFromPackageConfig, mbErr)
 import DA.Cli.Damlc.Test (CoveragePaths(..),
                           LoadCoverageOnly(..),
                           RunAllTests(..),
@@ -825,22 +824,8 @@ installDepsAndInitPackageDb opts (InitPkgDb shouldInit) =
         isProject <- withPackageConfig defaultProjectPath (const $ pure True) `catch` (\(_ :: ConfigError) -> pure False)
         when isProject $ do
             projRoot <- getCurrentDirectory
-            withPackageConfig defaultProjectPath $ \PackageConfigFields {..} -> do
-              damlAssistantIsSet <- damlAssistantIsSet
-              releaseVersion <- if damlAssistantIsSet
-                  then do
-                    damlPath <- getDamlPath
-                    damlEnv <- getDamlEnv damlPath (LookForProjectPath False)
-                    wrapErr "installing dependencies and initializing package database" $
-                      resolveReleaseVersionUnsafe (envUseCache damlEnv) pSdkVersion
-                  else pure (unsafeResolveReleaseVersion pSdkVersion)
-              installDependencies
-                  (toNormalizedFilePath' projRoot)
-                  opts
-                  releaseVersion
-                  pDependencies
-                  pDataDependencies
-              createProjectPackageDb (toNormalizedFilePath' projRoot) opts pModulePrefixes
+            withPackageConfig defaultProjectPath $
+              setupPackageDbFromPackageConfig (toNormalizedFilePath' projRoot) opts
 
 getMultiPackagePath :: MultiPackageLocation -> Maybe FilePath -> IO (Maybe ProjectPath)
 getMultiPackagePath multiPackageLocation searchPath =
@@ -1425,8 +1410,7 @@ execDocTest opts scriptDar (ImportSource importSource) files =
             wrapErr "running doc test" $
               resolveReleaseVersionUnsafe (envUseCache damlEnv) SdkVersion.Class.unresolvedBuiltinSdkVersion
           else pure (unsafeResolveReleaseVersion SdkVersion.Class.unresolvedBuiltinSdkVersion)
-      installDependencies "." opts releaseVersion [scriptDar] []
-      createProjectPackageDb "." opts mempty
+      setupPackageDb "." opts releaseVersion [scriptDar] [] mempty
 
       opts <- pure opts
         { optPackageDbs = projectPackageDatabase : optPackageDbs opts

sdk/compiler/damlc/lib/DA/Cli/Damlc/DependencyDb.hs

@@ -8,6 +8,7 @@ module DA.Cli.Damlc.DependencyDb
     , mainMarker
     , depMarker
     , dataDepMarker
+    , packageDbLockPath
     ) where
 
 import qualified "zip-archive" Codec.Archive.Zip as ZipArchive
@@ -93,8 +94,11 @@ dependenciesDir opts projRoot =
     fromNormalizedFilePath projRoot </> projectDependenciesDatabase </>
     lfVersionString (optDamlLfVersion opts)
 
-lockFile :: FilePath
-lockFile = "daml.lock"
+packageDbLockPath :: NormalizedFilePath -> FilePath
+packageDbLockPath projRoot = fromNormalizedFilePath projRoot </> damlArtifactDir </> "setup-package-db.lock"
+
+remotePackagesLockFile :: FilePath
+remotePackagesLockFile = "daml.lock"
 
 fingerprintFile :: FilePath
 fingerprintFile = "fingerprint.json"
@@ -338,11 +342,11 @@ resolvePkgs projRoot opts pkgs
                 | FullPkgName {pkgName, pkgVersion} <- missing
                 ]
             Right result -> do
-              writeLockFile lockFile result
+              writeLockFile remotePackagesLockFile result
               pure result
   where
     depsDir = dependenciesDir opts projRoot
-    lockFp = fromNormalizedFilePath projRoot </> lockFile
+    lockFp = fromNormalizedFilePath projRoot </> remotePackagesLockFile
 
 writeLockFile :: FilePath -> M.Map FullPkgName LF.PackageId -> IO ()
 writeLockFile lockFp resolvedPkgs = do

sdk/compiler/damlc/lib/DA/Cli/Damlc/Packaging.hs

@@ -5,6 +5,8 @@
 
 module DA.Cli.Damlc.Packaging
   ( createProjectPackageDb
+  , setupPackageDb
+  , setupPackageDbFromPackageConfig
   , mbErr
   , getUnitId
 
@@ -43,27 +45,35 @@ import qualified Module as GHC
 import qualified "ghc-lib-parser" Packages as GHC
 import System.Directory.Extra (copyFile, createDirectoryIfMissing, listFilesRecursive, removePathForcibly)
 import System.Exit
+import System.FileLock
 import System.FilePath
 import System.IO.Extra (hFlush, hPutStrLn, stderr, writeFileUTF8)
 import System.Info.Extra
 import System.Process (callProcess)
 import "ghc-lib-parser" UniqSet
 
 import DA.Bazel.Runfiles
+import DA.Cli.Damlc.DependencyDb
+import DA.Daml.Assistant.Env (getDamlEnv, getDamlPath, envUseCache)
+import DA.Daml.Assistant.Types (LookForProjectPath (..))
+import DA.Daml.Assistant.Util (wrapErr)
+import DA.Daml.Assistant.Version (resolveReleaseVersionUnsafe)
 import DA.Daml.Compiler.Dar
 import DA.Daml.Compiler.DataDependencies as DataDeps
 import DA.Daml.Compiler.DecodeDar (DecodedDalf(..), decodeDalf)
 import DA.Daml.Compiler.Output
-import qualified DA.Daml.LF.Ast as LF
 import DA.Daml.LF.Ast.Optics (packageRefs)
-import qualified DA.Daml.LFConversion.MetadataEncoding as LFC
 import DA.Daml.Options.Packaging.Metadata
 import DA.Daml.Options.Types
-import DA.Cli.Damlc.DependencyDb
-import qualified DA.Pretty
-import qualified DA.Service.Logger as Logger
+import DA.Daml.Package.Config (PackageConfigFields (..))
+import DA.Daml.Project.Consts (damlAssistantIsSet)
+import DA.Daml.Project.Types (ReleaseVersion, unsafeResolveReleaseVersion)
 import Development.IDE.Core.IdeState.Daml
 import Development.IDE.Core.RuleTypes.Daml
+import qualified DA.Daml.LF.Ast as LF
+import qualified DA.Daml.LFConversion.MetadataEncoding as LFC
+import qualified DA.Pretty
+import qualified DA.Service.Logger as Logger
 import SdkVersion.Class (SdkVersioned, damlStdlib)
 
 -- | Create the project package database containing the given dar packages.
@@ -945,3 +955,58 @@ prefixModules prefixes dalfs = do
                 ( prefix
                 , NM.names . LF.packageModules . LF.extPackagePkg $ LF.dalfPackagePkg pkg
                 )
+
+unsafeSetupPackageDb
+    :: SdkVersioned
+    => NormalizedFilePath
+    -> Options
+    -> ReleaseVersion
+    -> [String] -- Package dependencies. Can be base-packages, sdk-packages or filepath.
+    -> [FilePath] -- Data Dependencies. Can be filepath to dars/dalfs.
+    -> MS.Map UnitId GHC.ModuleName
+    -> IO ()
+unsafeSetupPackageDb projRoot opts releaseVersion pDependencies pDataDependencies pModulePrefixes = do
+    installDependencies
+        projRoot
+        opts
+        releaseVersion
+        pDependencies
+        pDataDependencies
+    createProjectPackageDb projRoot opts pModulePrefixes
+
+withPkgDbLock :: NormalizedFilePath -> IO a -> IO a
+withPkgDbLock projRoot act = do
+    let packageDbLockFile = packageDbLockPath projRoot
+    createDirectoryIfMissing True $ takeDirectory packageDbLockFile
+    withFileLock packageDbLockFile Exclusive $ const act
+
+-- Installs dependencies and creates package Db ready to be used
+setupPackageDb
+    :: SdkVersioned
+    => NormalizedFilePath
+    -> Options
+    -> ReleaseVersion
+    -> [String] -- Package dependencies. Can be base-packages, sdk-packages or filepath.
+    -> [FilePath] -- Data Dependencies. Can be filepath to dars/dalfs.
+    -> MS.Map UnitId GHC.ModuleName
+    -> IO ()
+setupPackageDb projRoot opts releaseVersion pDependencies pDataDependencies pModulePrefixes =
+    withPkgDbLock projRoot $ unsafeSetupPackageDb projRoot opts releaseVersion pDependencies pDataDependencies pModulePrefixes
+
+setupPackageDbFromPackageConfig
+    :: SdkVersioned
+    => NormalizedFilePath
+    -> Options
+    -> PackageConfigFields
+    -> IO ()
+setupPackageDbFromPackageConfig projRoot opts PackageConfigFields {..} =
+    withPkgDbLock projRoot $ do
+        damlAssistantIsSet <- damlAssistantIsSet
+        releaseVersion <- if damlAssistantIsSet
+            then do
+              damlPath <- getDamlPath
+              damlEnv <- getDamlEnv damlPath (LookForProjectPath False)
+              wrapErr "installing dependencies and initializing package database" $
+                resolveReleaseVersionUnsafe (envUseCache damlEnv) pSdkVersion
+            else pure (unsafeResolveReleaseVersion pSdkVersion)
+        unsafeSetupPackageDb projRoot opts releaseVersion pDependencies pDataDependencies pModulePrefixes

sdk/compiler/damlc/tests/src/DA/Test/DamlcIntegration.hs

@@ -94,8 +94,7 @@ import Test.Tasty.Options
 import Test.Tasty.Providers
 import Test.Tasty.Runners (Result(..))
 
-import DA.Cli.Damlc.DependencyDb (installDependencies)
-import DA.Cli.Damlc.Packaging (createProjectPackageDb)
+import DA.Cli.Damlc.Packaging (setupPackageDb)
 import Module (stringToUnitId)
 import SdkVersion (SdkVersioned, withSdkVersions, sdkVersion, sdkPackageVersion)
 
@@ -152,15 +151,12 @@ withVersionedDamlScriptDep packageFlagName darPath mLfVer extraPackages cont = d
 
       extraDars <- traverse (\(darName, _, _) -> locateRunfiles $ mainWorkspace </> "compiler" </> "damlc" </> "tests" </> darName <> ".dar") extraPackages
 
-      installDependencies
+      setupPackageDb
         projDir
         (defaultOptions mLfVer)
         (unsafeResolveReleaseVersion (either throw id (parseUnresolvedVersion (T.pack sdkVersion))))
         ["daml-prim", "daml-stdlib", scriptDar]
         extraDars
-      createProjectPackageDb
-        projDir
-        (defaultOptions mLfVer)
         mempty
 
       cont (dir </> projectPackageDatabase, packageFlags)

sdk/compiler/damlc/tests/src/DA/Test/ScriptService.hs

@@ -9,7 +9,6 @@ import Control.Exception
 import Control.Monad
 import DA.Bazel.Runfiles
 import DA.Cli.Damlc.Packaging
-import DA.Cli.Damlc.DependencyDb
 import qualified DA.Daml.LF.Ast.Version as LF
 import DA.Daml.LF.PrettyScript (prettyScriptError, prettyScriptResult)
 import qualified DA.Daml.LF.ScriptServiceClient as SS
@@ -97,19 +96,8 @@ withScriptService lfVersion action =
             "- daml-stdlib",
             "- " <> show scriptDar
           ]
-      withPackageConfig (ProjectPath ".") $ \PackageConfigFields {..} -> do
-        let projDir = toNormalizedFilePath' dir
-        installDependencies
-            projDir
-            (options lfVersion)
-            (unsafeResolveReleaseVersion pSdkVersion)
-            pDependencies
-            pDataDependencies
-        createProjectPackageDb
-          projDir
-          (options lfVersion)
-          pModulePrefixes
-
+      withPackageConfig (ProjectPath ".") $
+        setupPackageDbFromPackageConfig (toNormalizedFilePath' dir) $ options lfVersion
       logger <- Logger.newStderrLogger Logger.Debug "script-service"
 
       -- Spinning up the script service is expensive so we do it once at the beginning.

sdk/compiler/damlc/tests/src/DA/Test/ScriptService_1_15.hs

@@ -9,7 +9,6 @@ import Control.Exception
 import Control.Monad
 import DA.Bazel.Runfiles
 import DA.Cli.Damlc.Packaging
-import DA.Cli.Damlc.DependencyDb
 import qualified DA.Daml.LF.Ast.Version as LF
 import DA.Daml.LF.PrettyScenario (prettyScenarioError, prettyScenarioResult)
 import qualified DA.Daml.LF.ScriptServiceClient as SS
@@ -65,18 +64,8 @@ main = withSdkVersions $ do
             "- daml-stdlib",
             "- " <> show scriptDar
           ]
-      withPackageConfig (ProjectPath ".") $ \PackageConfigFields {..} -> do
-        let projDir = toNormalizedFilePath' dir
-        installDependencies
-            projDir
-            options
-            (unsafeResolveReleaseVersion pSdkVersion)
-            pDependencies
-            pDataDependencies
-        createProjectPackageDb
-          projDir
-          options
-          pModulePrefixes
+      withPackageConfig (ProjectPath ".") $
+        setupPackageDbFromPackageConfig (toNormalizedFilePath' dir) options
 
       logger <- Logger.newStderrLogger Logger.Debug "script-service"