Skip to content

Commit 1b4e45e

Browse files
MpdreamzMpdreamz
committed
Improvement/build (#2523)
* Several build related fixes - Normalize module vs type usages in fsx files - unit methods now all in Method() = form - cleaned up build.bat and its skiptests leniency, now only accepted as the first argument - Improved usage notice in build.bat - Building now checks if `dotnet.exe` resolves to using the proper SDK - Moved to DotNetCli FAKE helper (#2395) - Separated Projects and Tooling out of Paths.fsx - Now have a truly quick compile target `build quick [test_filter]` does an incremental build and runs the unit tests from the bin output folder. it also skips paket. - moved global.json to the root of repos * Explict separation of unit/integration custom runners - unit tests now parallizes over all tests ignoring all clusters rather then piggy backing on xunit which we do in 5.x (our own routine is 10+seconds faster) or on the integration routine (which we currently do in master) * moved testing over to DotNetCli helper too * Build resiliency changes build quick can optionally does a paket bootstrap/restore if paket is not found, allowing you to do build quick on a clean checkout too. The general build now cleans the bin folders after itself so that running build build quick Does not end up in the latter resolving 4.6 assemblies from the previous build Conflicts: build.bat build/scripts/Building.fsx build/scripts/Paths.fsx build/scripts/Targets.fsx build/scripts/Testing.fsx paket.lock src/Benchmarking/Benchmarking.csproj src/CodeGeneration/ApiGenerator/ApiGenerator.csproj src/CodeGeneration/DocGenerator/DocGenerator.csproj src/Nest/Nest.csproj src/Profiling/Profiling.csproj src/Tests/Framework/Xunit/TestAssemblyRunner.cs src/Tests/Tests.csproj
1 parent c4b6d28 commit 1b4e45e

File tree

24 files changed

+1271
-1666
lines changed

24 files changed

+1271
-1666
lines changed

‎build.bat‎

Lines changed: 49 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,24 @@
11
@echo off
22

3-
REM build
4-
REM build build [skiptests]
5-
REM build release [version] [skiptests]
6-
REM build version [version] [skiptests]
7-
REM build integrate [elasticsearch_versions] [clustername]
8-
REM build canary [apikey] [feed] [skiptests]
9-
10-
REM - elasticsearch_versions can be multiple separated with a semi-colon ';'
11-
12-
.paket\paket.bootstrapper.exe
13-
if errorlevel 1 (
14-
exit /b %errorlevel%
15-
)
16-
.paket\paket.exe restore
17-
if errorlevel 1 (
18-
exit /b %errorlevel%
19-
)
3+
REM USAGE:
4+
REM
5+
REM build [skiptests] <command> [params]
6+
REM
7+
REM COMMANDS:
8+
REM
9+
REM * build
10+
REM default target if non provided. Performs a clean, rebuild and test of all target frameworks
11+
REM * quick [testfilter]
12+
REM incremental build and unit test for .NET 4.5, [testfilter] allows you to do
13+
REM a contains match on the tests to be run.
14+
REM * build release <version>
15+
REM create a release worthy nuget packages for [version] under build\output
16+
REM * build integrate <elasticsearch_versions> [clustername] [testfilter] -
17+
REM run integration tests for <elasticsearch_versions> which is a semicolon separated list of
18+
REM elasticsearch versions to test or `latest`. Can filter tests by <clustername> and <testfilter>
19+
REM * build canary [apikey] [feed] [skiptests]
20+
REM create a canary nuget package based on the current version if [feed] and [apikey] are provided
21+
REM also pushes to upstream (myget)
2022

2123
SET TARGET="build"
2224
SET VERSION=
@@ -32,36 +34,49 @@ IF /I "%1"=="skiptests" (
3234
set SKIPTESTS="1"
3335
SHIFT
3436
)
37+
IF NOT [%1]==[] (set TARGET=%1)
3538

36-
IF NOT [%1]==[] (set TARGET="%1")
39+
SET SKIPPAKET=0
40+
IF /I "%TARGET%"=="quick" SET SKIPPAKET=1
41+
IF /I "%TARGET%"=="forever" SET SKIPPAKET=1
3742

38-
IF /I "%1"=="version" (
43+
IF "%SKIPPAKET%" neq "1" (
44+
.paket\paket.bootstrapper.exe
45+
.paket\paket.exe restore
46+
)
47+
48+
REM if `build quick` is called on a fresh checkout force a restore anyway
49+
IF "%SKIPPAKET%"=="1" (
50+
IF NOT EXIST .paket\paket.exe (
51+
.paket\paket.bootstrapper.exe
52+
.paket\paket.exe restore
53+
)
54+
)
55+
56+
IF /I "%TARGET%"=="version" (
3957
IF NOT [%2]==[] (set VERSION="%2")
40-
IF /I "%3"=="skiptests" (set SKIPTESTS=1)
41-
IF /I "%2"=="skiptests" (set SKIPTESTS=1)
4258
)
43-
IF /I "%1"=="release" (
59+
IF /I "%TARGET%"=="release" (
4460
IF NOT [%2]==[] (set VERSION="%2")
45-
IF /I "%3"=="skiptests" (set SKIPTESTS=1)
46-
IF /I "%2"=="skiptests" (set SKIPTESTS=1)
4761
)
48-
49-
IF /I "%1%"=="integrate" (
62+
IF /I "%TARGET%"=="quick" (
63+
IF NOT [%2]==[] (set NEST_TEST_FILTER="%2")
64+
)
65+
IF /I "%TARGET%"=="forever" (
66+
IF NOT [%2]==[] (set NEST_TEST_FILTER="%2")
67+
)
68+
IF /I "%TARGET%"=="integrate" (
5069
IF NOT [%2]==[] (set ESVERSIONS="%2")
5170
IF NOT [%3]==[] (set NEST_INTEGRATION_CLUSTER="%3")
5271
IF NOT [%4]==[] (set NEST_TEST_FILTER="%4")
5372
)
54-
55-
IF /I "%1%"=="canary" (
56-
IF NOT [%2]==[] IF NOT "%2"=="skiptests" (
73+
IF /I "%TARGET%"=="canary" (
74+
IF NOT [%2]==[] (
5775
set APIKEY="%2"
5876
SET APIKEYPROVIDED="<redacted>"
5977
)
60-
IF NOT [%3]==[] IF NOT "%3"=="skiptests" set FEED="%3"
61-
IF /I "%4"=="skiptests" (set SKIPTESTS=1)
62-
IF /I "%3"=="skiptests" (set SKIPTESTS=1)
63-
IF /I "%2"=="skiptests" (set SKIPTESTS=1)
78+
IF NOT [%3]==[] set FEED="%3"
6479
)
6580

66-
ECHO starting build using target=%TARGET% version=%VERSION% esversions=%ESVERSIONS% skiptests=%SKIPTESTS% apiKey=%APIKEYPROVIDED% feed=%FEED% escluster=%NEST_INTEGRATION_CLUSTER% testfilter=%NEST_TEST_FILTER%
81+
ECHO build.bat: target=%TARGET% skippakket=%SKIPPAKET% version=%VERSION% esversions=%ESVERSIONS% skiptests=%SKIPTESTS% apiKey=%APIKEYPROVIDED% feed=%FEED% escluster=%NEST_INTEGRATION_CLUSTER% testfilter=%NEST_TEST_FILTER%
6782
"packages\build\FAKE\tools\Fake.exe" "build\\scripts\\Targets.fsx" "target=%TARGET%" "version=%VERSION%" "esversions=%ESVERSIONS%" "skiptests=%SKIPTESTS%" "apiKey=%APIKEY%" "feed=%FEED%" "escluster=%NEST_INTEGRATION_CLUSTER%" "testfilter=%NEST_TEST_FILTER%"

‎build/scripts/Building.fsx‎

Lines changed: 61 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -1,61 +1,90 @@
11
#I @"../../packages/build/FAKE/tools"
2+
#I @"../../packages/build/FSharp.Data/lib/net40"
23
#r @"FakeLib.dll"
4+
#r @"FSharp.Data.dll"
35

46
#load @"Paths.fsx"
7+
#load @"Tooling.fsx"
58

9+
open System
610
open Fake
11+
open FSharp.Data
712

813
open Paths
914
open Projects
10-
open Tooling;
15+
open Tooling
1116

12-
type Build() =
17+
module Build =
1318

14-
static let runningRelease = hasBuildParam "version" || hasBuildParam "apikey" || getBuildParam "target" = "canary" || getBuildParam "target" = "release"
19+
let private runningRelease = hasBuildParam "version" || hasBuildParam "apikey" || getBuildParam "target" = "canary" || getBuildParam "target" = "release"
20+
let private quickBuild = getBuildParam "target" = "quick" || getBuildParam "target" = "forever"
21+
22+
type private GlobalJson = JsonProvider<"../../global.json">
23+
let private pinnedSdkVersion = GlobalJson.GetSample().Sdk.Version
24+
25+
let private compileCore() =
26+
if not (DotNetCli.isInstalled()) then failwith "You need to install the dotnet command line SDK to build for .NET Core"
27+
let runningSdkVersion = DotNetCli.getVersion()
28+
if (runningSdkVersion <> pinnedSdkVersion) then failwithf "Attempting to run with dotnet.exe with %s but global.json mandates %s" runningSdkVersion pinnedSdkVersion
1529

16-
static let compileCore() =
1730
DotNetProject.AllPublishable
1831
|> Seq.iter(fun p ->
1932
let path = Paths.ProjectJson p.Name
2033
let o = Paths.ProjectOutputFolder p DotNetFramework.NetStandard1_3
21-
DotNet.Exec ["restore"; path; "--verbosity Warning"]
22-
DotNet.Exec ["build"; path; "--configuration Release"; "-o"; o; "-f"; DotNetFramework.NetStandard1_3.Identifier.MSBuild]
34+
DotNetCli.Restore
35+
(fun p ->
36+
{ p with
37+
Project = path
38+
TimeOut = TimeSpan.FromMinutes(2.)
39+
}
40+
) |> ignore
41+
42+
DotNetCli.Build
43+
(fun p ->
44+
{ p with
45+
Configuration = "Release"
46+
Project = path
47+
Framework = DotNetFramework.NetStandard1_3.Identifier.MSBuild
48+
TimeOut = TimeSpan.FromMinutes(2.)
49+
AdditionalArgs = ["-o"; o]
50+
}
51+
) |> ignore
52+
2353
)
2454

25-
static let compileDesktop target =
26-
MsBuild.Build(target, DotNetFramework.Net45.Identifier)
27-
MsBuild.Build(target, DotNetFramework.Net46.Identifier)
55+
let private compileDesktop() =
56+
Tooling.MsBuild.Rebuild DotNetFramework.Net45.Identifier
57+
Tooling.MsBuild.Rebuild DotNetFramework.Net46.Identifier
2858

29-
static let gitLink() =
59+
let private gitLink() =
3060
DotNetProject.AllPublishable
3161
|> Seq.iter(fun p ->
3262
let projectName = (p.Name |> directoryInfo).Name
3363
let link framework =
34-
//TC sets these so gitlink does not need to recreate
35-
let branch = environVarOrNone "GITBRANCH"
36-
let commit = environVarOrNone "GITCOMMIT"
37-
let args = Seq.empty
38-
|> match branch with
39-
| Some b -> Seq.append ["-b"; b]
40-
| None -> Seq.append Seq.empty
41-
|> match commit with
42-
| Some c -> Seq.append ["-s"; c]
43-
| None -> Seq.append Seq.empty
44-
|> Seq.append ["."; "-u"; Paths.Repository; "-d"; (Paths.ProjectOutputFolder p framework); "-include"; projectName]
45-
GitLink.Exec ["."; "-u"; Paths.Repository; "-d"; (Paths.ProjectOutputFolder p framework); "-include"; projectName]
64+
Tooling.GitLink.Exec ["."; "-u"; Paths.Repository; "-d"; (Paths.ProjectOutputFolder p framework); "-include"; projectName]
4665
|> ignore
4766
link DotNetFramework.Net45
4867
link DotNetFramework.Net46
4968
link DotNetFramework.NetStandard1_3
5069
)
5170

52-
static member Compile() =
53-
//let target = if runningRelease then "Rebuild" else "Build"
54-
compileDesktop "Rebuild"
55-
//we only need this output when doing a release otherwise depend on test to validate the build
56-
if runningRelease then compileCore()
57-
if not isMono && runningRelease then gitLink()
58-
59-
static member Clean() =
60-
CleanDir Paths.BuildOutput
61-
DotNetProject.All |> Seq.iter(fun p -> CleanDir(Paths.BinFolder p.Name))
71+
let Compile() =
72+
match quickBuild with
73+
| true -> Tooling.MsBuild.Build DotNetFramework.Net45.Identifier
74+
| false ->
75+
compileDesktop()
76+
compileCore()
77+
if not isMono && runningRelease then gitLink()
78+
79+
let Clean() =
80+
match quickBuild with
81+
| true -> ignore()
82+
| false ->
83+
CleanDir Paths.BuildOutput
84+
DotNetProject.All |> Seq.iter(fun p -> CleanDir(Paths.BinFolder p.Name))
85+
86+
let CleanAfter() =
87+
match quickBuild with
88+
| true -> ignore()
89+
| false ->
90+
DotNetProject.All |> Seq.iter(fun p -> CleanDir(Paths.BinFolder p.Name))

‎build/scripts/Documentation.fsx‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ open System.IO
1212

1313
module Documentation =
1414

15-
let Generate() =
15+
let Generate() =
1616
let generator = "build/output/v4.6/DocGenerator/DocGenerator.exe"
1717
ExecProcess (fun p ->
1818
p.WorkingDirectory <- "src/CodeGeneration/DocGenerator"
@@ -21,7 +21,7 @@ module Documentation =
2121
(TimeSpan.FromMinutes (1.0)) |> ignore
2222

2323
// TODO: hook documentation validation into the process
24-
let Validate() =
24+
let Validate() =
2525
let elasticDocsDir = "../elasticsearch-docs"
2626
if (directoryExists elasticDocsDir = false) then
2727
let fullPath = combinePaths currentDirectory elasticDocsDir |> Path.GetFullPath

0 commit comments

Comments
 (0)