Replace use of TH in readJSONFromFile with generated haskell modules

cardano-constitution/src/Cardano/Constitution/Config.hs

@@ -19,7 +19,7 @@ import Data.Aeson.THReader as Aeson
 
 -- | The default config read from "data/defaultConstitution.json"
 defaultConstitutionConfig :: ConstitutionConfig
-defaultConstitutionConfig = $$(Aeson.readJSONFromFile DFP.defaultConstitutionConfigFile)
+defaultConstitutionConfig = Aeson.readJSONFromFile DFP.defaultConstitutionConfigFile
 {-# INLINEABLE defaultConstitutionConfig #-}
 
 -- | NOTE: **BE CAREFUL** of the ordering. Expected value is first arg, Proposed Value is second arg

cardano-constitution/test/Cardano/Constitution/Config/Tests.hs

@@ -19,8 +19,7 @@ import Helpers.TestBuilders
 import Test.Tasty.QuickCheck
 
 defaultConstitutionJSONSchema :: Aeson.Value
-defaultConstitutionJSONSchema =
-  $$(Aeson.readJSONFromFile DFP.defaultConstitutionJSONSchemaFile)
+defaultConstitutionJSONSchema = Aeson.readJSONFromFile DFP.defaultConstitutionJSONSchemaFile
 
 {-| All the examples in the JSON schema are parseable as a list of ConstitutionConfigs.
 Actually the examples 9005 and 9006 should not normally parse,

plutus-core/cost-model/CostModelGeneration.md

@@ -40,17 +40,17 @@ costing functions involves a number of steps.
   when testing costing benchmarks.
 
 * Change directory to `plutus-core/cost-model/data/` and run `cabal run
-  plutus-core:generate-cost-model -- --csv <file>`, where `<file>` is the CSV
-  file produced in the previous step.  This runs some R code in
-  [`plutus-core/cost-model/data/models.R`](./data/models.R) which fits a linear
+  plutus-core:generate-cost-model -- --csv <file> -o <output.hs>`, where `<file>` is the CSV
+  file produced in the previous step and `<output.hs>` is the Haskell module file to generate.
+  This runs some R code in [`plutus-core/cost-model/data/models.R`](./data/models.R) which fits a linear
   model to the data for each builtin; the general form of the model for each
   builtin is coded into `models.R`. Certain checks are performed during this
   process: for example it is possible that R will generate a model with a
   negative coefficient (for example if the results for a builtin are roughly
   constant) and if that happens then a warning is printed and the coefficient is
   replaced by zero.
 
-  * The output of `generate-cost-model` is a JSON object describing the form of
+  * The output of `generate-cost-model` is a Haskell module file describing the form of
   the models for each builtin, together with the model coefficients fitted by R.
   By default this is written to the terminal, but an output file can be
   specified with `-o`.  The model coefficients are converted from floating point
@@ -60,23 +60,23 @@ costing functions involves a number of steps.
   on different machines).
 
 * The specific cost model data to be used by the Plutus Core evaluator should be
-  checked in to git in the file
-  [`plutus-core/cost-model/data/builtinCostModelC.json`](./data/builtinCostModelC.json).
-  There are also files called `builtinCostModelA.json` and
-  `builtinCostModelB.json` which are used for evaluating scripts prior to the
-  Chang hard fork: data for new builtins can (if fact, must) be added to these
-  files, but the existing content must not be changed.  The CSV file containing
-  the benchmark results used to generate the cost model should be checked in to
-  the repository; this is not strictly necessary but it can be useful to have
-  the raw data available if the details of the cost model need to be looked at
-  at some later time.  The benchmarking results used to generate the current cost
-  model (March 2025) are checked in in
+  checked in to git as Haskell modules in the directory
+  [`plutus-core/plutus-core/src/PlutusCore/Evaluation/Machine/CostModel/Generated/`](../plutus-core/src/PlutusCore/Evaluation/Machine/CostModel/Generated/).
+  The module `BuiltinCostModelC.hs` contains the cost model for the latest version.
+  There are also modules called `BuiltinCostModelA.hs` and `BuiltinCostModelB.hs`
+  which are used for evaluating scripts prior to the Chang hard fork: data for new
+  builtins can (if fact, must) be added to these files, but the existing content
+  must not be changed.  The CSV file containing the benchmark results used to generate
+  the cost model should be checked in to the repository; this is not strictly necessary
+  but it can be useful to have the raw data available if the details of the cost model
+  need to be looked at at some later time.  The benchmarking results used to generate
+  the current cost model (March 2025) are checked in in
   [`plutus-core/cost-model/data/benching-conway.csv`](./data/benching-conway.csv)
   and any new results should be added to the end of that file.
 
-* When the rest of the `plutus-core` package is compiled, the contents of
-  `builtCostModelC.json` are read and used by some Template Haskell code to
-  construct Haskell functions which implement the cost models.
+* When the rest of the `plutus-core` package is compiled, the generated Haskell
+  modules are directly imported and used to construct the cost models. This
+  eliminates the need for Template Haskell file reading at compile time.
 
 * To ensure consistency, `cabal bench plutus-core:cost-model-test` runs some
   QuickCheck tests to run the R models and the Haskell models and checks that
@@ -96,8 +96,8 @@ costing functions involves a number of steps.
   predicted by the builtin cost model, and divide the remaining time
   by the number of basic machine steps executed to arrive at an
   average time for each machine step (see the earlier discussion).
-  This is then stored in another JSON file,
-  [`plutus-core/cost-model/data/cekMachineCosts.json`](./data/cekMachineCosts.json).
+  This is then stored in Haskell modules in the `CostModel/Generated/` directory,
+  such as [`CekMachineCostsC.hs`](../plutus-core/src/PlutusCore/Evaluation/Machine/CostModel/Generated/CekMachineCostsC.hs).
   This cost is currently the same for each step, but more careful
   testing may enable us to produce more precise costs per step at some
   future date.  The JSON file also contains a constant cost for

plutus-core/cost-model/create-cost-model/Main.hs

@@ -1,19 +1,27 @@
+{-# LANGUAGE OverloadedStrings #-}
+
 module Main where
 
 import CreateBuiltinCostModel (createBuiltinCostModel)
-
-import Data.Aeson.Encode.Pretty
-import Data.ByteString.Lazy qualified as BSL (ByteString, putStr, writeFile)
+import PlutusCore.Evaluation.Machine.BuiltinCostModel (BuiltinCostModel)
+
+import Data.Aeson
+import Data.Aeson.Encode.Pretty (Config (..), defConfig, encodePretty')
+import Data.ByteString.Lazy.Char8 qualified as BSL
+import Data.Char (toLower)
+import Data.Text qualified as T
+import Data.Text.IO qualified as T
 import Options.Applicative
 import System.Directory
 import System.Exit
 import System.IO (hPutStrLn, stderr)
+import Text.Show.Pretty (ppShow)
 
 import Language.R (defaultConfig, runRegion, withEmbeddedR)
 
 {-| This takes a CSV file of benchmark results for built-in functions, runs the R
    code in `models.R` to construct costing functions based on the benchmark
-   results, and then produces JSON output containing the types and coefficients
+   results, and then produces Haskell module output containing the types and coefficients
    of the costing functions. For best results, run this in
    `plutus-core/cost-model/data` to make `models.R` easy to find; if that's
    inconvenient for some reason, use the `-m` option to provide a path to
@@ -36,7 +44,7 @@ data RFile = RFile FilePath
 defaultRFile :: RFile
 defaultRFile = RFile "models.R"
 
--- | Where to write the JSON output, stdout by default
+-- | Where to write the Haskell module output, stdout by default
 data Output = NamedOutput FilePath | StdOutput
 
 ---------------- Option parsers ----------------
@@ -80,8 +88,8 @@ fileOutput =
     <$> strOption
       ( long "output"
           <> short 'o'
-          <> metavar "FILENAME"
-          <> help "Output file"
+          <> metavar "MODULENAME"
+          <> help "Output Haskell module name, e.g. BuiltinCostModelA, will be placed under PlutusCore/Evaluation/Machine/CostModel/Generated"
       )
 
 stdOutput :: Parser Output
@@ -102,7 +110,7 @@ arguments =
     ( fullDesc
         <> header "Plutus Core cost model creation tool"
         <> progDesc
-          ( "Creates a JSON description of Plutus Core cost model "
+          ( "Creates a Haskell module containing the Plutus Core cost model "
               ++ "for built-in functions from a set of benchmark results "
               ++ "produced by cost-model-budgeting-bench"
           )
@@ -145,17 +153,50 @@ checkBenchmarkFile file =
           ++ "The default results file is plutus-core/cost-model/data/benching.csv."
    in checkInputFile file "benchmark results file" advice
 
-writeOutput
-  :: Output -> BSL.ByteString -> IO ()
-writeOutput outp v = do
+writeOutput :: Output -> BuiltinCostModel -> IO ()
+writeOutput outp model = do
   case outp of
-    NamedOutput file -> BSL.writeFile file v
-    StdOutput -> BSL.putStr v
+    NamedOutput moduleName -> do
+      let (modulePath, moduleCode) = generateCostModelHaskellModule moduleName model
+      T.writeFile modulePath moduleCode
+    StdOutput ->
+      BSL.putStr modelJson
+  where
+    modelJson = encodePretty' (defConfig {confCompare = \_ _ -> EQ}) model
+
+generateCostModelHaskellModule :: String -> BuiltinCostModel -> (FilePath, T.Text)
+generateCostModelHaskellModule moduleName model = (modulePath, moduleCode)
+  where
+    modulePath = "plutus-core/plutus-core/src/PlutusCore/Evaluation/Machine/CostModel/" <> moduleName <> ".hs"
+    modelId = T.pack (toLowerFirstCharOnly moduleName)
+    moduleCode =
+      T.unlines
+        [ "-- This file is auto-generated by the generate-cost-model executable."
+        , "-- Do not edit this file manually."
+        , ""
+        , "module PlutusCore.Evaluation.Machine.CostModel." <> T.pack moduleName <> " (" <> modelId <> ") where"
+        , ""
+        , "import PlutusCore.Evaluation.Machine.BuiltinCostModel"
+        , ""
+        , modelId <> " :: BuiltinCostModel"
+        , modelId <> " = " <> T.pack (ppShow model)
+        ]
+
+    toLowerFirstCharOnly :: String -> String
+    toLowerFirstCharOnly "" = ""
+    toLowerFirstCharOnly (c : cs) = toLower c : cs
 
 main :: IO ()
 main = do
   (BenchmarkFile bmfile, RFile rfile, out) <- execParser arguments
-  checkBenchmarkFile bmfile
-  checkRFile rfile
-  model <- withEmbeddedR defaultConfig $ runRegion $ createBuiltinCostModel bmfile rfile
-  writeOutput out $ encodePretty' (defConfig {confCompare = \_ _ -> EQ}) model
+  -- checkBenchmarkFile bmfile
+  -- checkRFile rfile
+  -- model <- withEmbeddedR defaultConfig $ runRegion $ createBuiltinCostModel bmfile rfile
+  -- writeOutput out model
+  case out of
+    NamedOutput moduleName -> do
+      model <- eitherDecodeFileStrict ("plutus-core/cost-model/data/builtinCostModelA.json")
+      case model of
+        Left err -> putStrLn err
+        Right model' -> writeOutput out model'
+    _ -> undefined

plutus-core/plutus-core.cabal

@@ -22,13 +22,8 @@ extra-doc-files:
 -- `data-files`.  See https://github.com/haskell/cabal/pull/6889 and the issue
 -- #4746 that it mentions.
 extra-source-files:
+  cost-model/data/*.csv
   cost-model/data/*.R
-  cost-model/data/builtinCostModelA.json
-  cost-model/data/builtinCostModelB.json
-  cost-model/data/builtinCostModelC.json
-  cost-model/data/cekMachineCostsA.json
-  cost-model/data/cekMachineCostsB.json
-  cost-model/data/cekMachineCostsC.json
   plutus-core/test/CostModelInterface/defaultCostModelParams.json
 
 source-repository head
@@ -136,6 +131,12 @@ library
     PlutusCore.Evaluation.Machine.CostingFun.Core
     PlutusCore.Evaluation.Machine.CostingFun.JSON
     PlutusCore.Evaluation.Machine.CostingFun.SimpleJSON
+    PlutusCore.Evaluation.Machine.CostModel.BuiltinCostModelA
+    PlutusCore.Evaluation.Machine.CostModel.BuiltinCostModelB
+    PlutusCore.Evaluation.Machine.CostModel.BuiltinCostModelC
+    PlutusCore.Evaluation.Machine.CostModel.CekMachineCostsA
+    PlutusCore.Evaluation.Machine.CostModel.CekMachineCostsB
+    PlutusCore.Evaluation.Machine.CostModel.CekMachineCostsC
     PlutusCore.Evaluation.Machine.CostModelInterface
     PlutusCore.Evaluation.Machine.CostStream
     PlutusCore.Evaluation.Machine.ExBudget
@@ -350,7 +351,6 @@ library
     , text
     , th-lift
     , th-lift-instances
-    , th-utilities
     , time
     , transformers
     , unordered-containers
@@ -992,6 +992,7 @@ executable generate-cost-model
     buildable: False
 
   build-depends:
+    , aeson
     , aeson-pretty
     , barbies
     , base                  >=4.9   && <5
@@ -1000,9 +1001,9 @@ executable generate-cost-model
     , inline-r              >=1.0.1
     , optparse-applicative
     , plutus-core           ^>=1.58
+    , pretty-show
     , text
 
-  --    , exceptions
   other-modules:
     BuiltinMemoryModels
     CreateBuiltinCostModel

plutus-core/plutus-core/src/Data/Aeson/THReader.hs

@@ -3,12 +3,11 @@
 module Data.Aeson.THReader where
 
 import Data.Aeson
-import Language.Haskell.TH.Syntax
-import TH.RelativePaths
+import System.IO.Unsafe
 
-readJSONFromFile :: (FromJSON a, Lift a) => String -> Code Q a
-readJSONFromFile name = liftCode $ do
-  contents <- qReadFileLBS name
-  case eitherDecode contents of
-    Left err -> fail err
-    Right res -> examineCode [||res||]
+{-# OPAQUE readJSONFromFile #-}
+readJSONFromFile :: FromJSON a => String -> a
+readJSONFromFile path =
+  case unsafePerformIO (eitherDecodeFileStrict path) of
+    Left err -> error ("Failed to decode json file " <> path <> ":\n" <> err)
+    Right res -> res

plutus-core/plutus-core/src/PlutusCore/DataFilePaths.hs

@@ -1,54 +1,19 @@
-{-| Various file paths used in plutus-core, currently all to do with the cost
-model. -}
-module PlutusCore.DataFilePaths
-where
+{-| Various file paths used in plutus-core.
+    These paths are primarily used for testing and benchmarking.
+    Cost models are embedded as Haskell modules in
+    PlutusCore.Evaluation.Machine.CostModel.* -}
+module PlutusCore.DataFilePaths where
 
 import System.FilePath
 
 costModelDataDir :: FilePath
 costModelDataDir = "cost-model" </> "data"
 
-builtinCostModelFileA :: FilePath
-builtinCostModelFileA = costModelDataDir </> "builtinCostModelA" <.> "json"
-
-builtinCostModelFileB :: FilePath
-builtinCostModelFileB = costModelDataDir </> "builtinCostModelB" <.> "json"
-
-builtinCostModelFileC :: FilePath
-builtinCostModelFileC = costModelDataDir </> "builtinCostModelC" <.> "json"
-
-builtinCostModelFileD :: FilePath
-builtinCostModelFileD = costModelDataDir </> "builtinCostModelB" <.> "json"
-
-builtinCostModelFileE :: FilePath
-builtinCostModelFileE = costModelDataDir </> "builtinCostModelC" <.> "json"
-
-latestBuiltinCostModelFile :: FilePath
-latestBuiltinCostModelFile = builtinCostModelFileC
-
-cekMachineCostsFileA :: FilePath
-cekMachineCostsFileA = costModelDataDir </> "cekMachineCostsA" <.> "json"
-
-cekMachineCostsFileB :: FilePath
-cekMachineCostsFileB = costModelDataDir </> "cekMachineCostsB" <.> "json"
-
-cekMachineCostsFileC :: FilePath
-cekMachineCostsFileC = costModelDataDir </> "cekMachineCostsC" <.> "json"
-
-cekMachineCostsFileD :: FilePath
-cekMachineCostsFileD = costModelDataDir </> "cekMachineCostsB" <.> "json"
-
-cekMachineCostsFileE :: FilePath
-cekMachineCostsFileE = costModelDataDir </> "cekMachineCostsC" <.> "json"
-
-latestMachineCostsFile :: FilePath
-latestMachineCostsFile = cekMachineCostsFileC
-
--- | The file containing the R models: only needed for cost-model-test.
+-- | The file containing the R models: needed for cost-model-test and generate-cost-model.
 rModelFile :: FilePath
 rModelFile = costModelDataDir </> "models" <.> "R"
 
-{-| The file containing the benchmark results for the built-in functions: only
-needed for cost-model-test. -}
+{-| The file containing the benchmark results for the built-in functions:
+needed for cost-model-test and generate-cost-model. -}
 benchingResultsFile :: FilePath
 benchingResultsFile = costModelDataDir </> "benching-conway" <.> "csv"