Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion Directory.Packages.props
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
</PropertyGroup>
<ItemGroup>
<PackageVersion Include="HtmlAgilityPack" Version="1.12.4" />
<PackageVersion Include="Microsoft.Playwright" Version="1.48.0" />
<PackageVersion Include="Microsoft.AspNetCore.Mvc.Testing" Version="9.0.0" />
<PackageVersion Include="Microsoft.Extensions.DependencyInjection" Version="9.0.0" />
<PackageVersion Include="Microsoft.Extensions.Logging" Version="9.0.0" />
Expand All @@ -13,4 +14,4 @@
<PackageVersion Include="xunit" Version="2.9.2" />
<PackageVersion Include="xunit.runner.visualstudio" Version="2.8.2" />
</ItemGroup>
</Project>
</Project>
15 changes: 15 additions & 0 deletions NetInteractor.sln
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NetInteractor.Test", "test\
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NetInteractor.PuppeteerSharp", "src\NetInteractor.PuppeteerSharp\NetInteractor.PuppeteerSharp.csproj", "{96C0F6DB-7F80-4E41-B9E8-75C5F750B4BF}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NetInteractor.Playwright", "src\NetInteractor.Playwright\NetInteractor.Playwright.csproj", "{1451EBF6-EF0F-4F7E-8F3E-7529AF5F9E62}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Expand Down Expand Up @@ -57,6 +59,18 @@ Global
{96C0F6DB-7F80-4E41-B9E8-75C5F750B4BF}.Release|x64.Build.0 = Release|Any CPU
{96C0F6DB-7F80-4E41-B9E8-75C5F750B4BF}.Release|x86.ActiveCfg = Release|Any CPU
{96C0F6DB-7F80-4E41-B9E8-75C5F750B4BF}.Release|x86.Build.0 = Release|Any CPU
{1451EBF6-EF0F-4F7E-8F3E-7529AF5F9E62}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{1451EBF6-EF0F-4F7E-8F3E-7529AF5F9E62}.Debug|Any CPU.Build.0 = Debug|Any CPU
{1451EBF6-EF0F-4F7E-8F3E-7529AF5F9E62}.Debug|x64.ActiveCfg = Debug|Any CPU
{1451EBF6-EF0F-4F7E-8F3E-7529AF5F9E62}.Debug|x64.Build.0 = Debug|Any CPU
{1451EBF6-EF0F-4F7E-8F3E-7529AF5F9E62}.Debug|x86.ActiveCfg = Debug|Any CPU
{1451EBF6-EF0F-4F7E-8F3E-7529AF5F9E62}.Debug|x86.Build.0 = Debug|Any CPU
{1451EBF6-EF0F-4F7E-8F3E-7529AF5F9E62}.Release|Any CPU.ActiveCfg = Release|Any CPU
{1451EBF6-EF0F-4F7E-8F3E-7529AF5F9E62}.Release|Any CPU.Build.0 = Release|Any CPU
{1451EBF6-EF0F-4F7E-8F3E-7529AF5F9E62}.Release|x64.ActiveCfg = Release|Any CPU
{1451EBF6-EF0F-4F7E-8F3E-7529AF5F9E62}.Release|x64.Build.0 = Release|Any CPU
{1451EBF6-EF0F-4F7E-8F3E-7529AF5F9E62}.Release|x86.ActiveCfg = Release|Any CPU
{1451EBF6-EF0F-4F7E-8F3E-7529AF5F9E62}.Release|x86.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
Expand All @@ -65,5 +79,6 @@ Global
{75781078-1761-4EDD-A872-5F71D1FF9721} = {F5108642-5386-4F95-ACF8-8DA0AD621820}
{A9186BD7-EFFC-4B4D-B828-C4F4AA924C50} = {F5108642-5386-4F95-ACF8-8DA0AD621820}
{96C0F6DB-7F80-4E41-B9E8-75C5F750B4BF} = {F5108642-5386-4F95-ACF8-8DA0AD621820}
{1451EBF6-EF0F-4F7E-8F3E-7529AF5F9E62} = {F5108642-5386-4F95-ACF8-8DA0AD621820}
EndGlobalSection
EndGlobal
22 changes: 22 additions & 0 deletions src/NetInteractor.Playwright/NetInteractor.Playwright.csproj
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFrameworks>netstandard2.0;net8.0;net9.0</TargetFrameworks>
<Authors>Kerry Jiang</Authors>
<Description>Playwright implementation of IWebAccessor for NetInteractor - enables browser automation with Chromium/Firefox/WebKit.</Description>
<PackageLicenseExpression>Apache-2.0</PackageLicenseExpression>
<PackageTags>Web;Automation;Html;Scraping;Testing;Playwright;Browser;WebAutomation;Headless;Chromium;Firefox;WebKit</PackageTags>
<RepositoryUrl>https://github.com/kerryjiang/NetInteractor.git</RepositoryUrl>
</PropertyGroup>
<PropertyGroup Condition="'$(IncludeReleaseNotes)' == 'true'">
<PackageReadmeFile>v$(PackageVersion).md</PackageReadmeFile>
</PropertyGroup>
<ItemGroup Condition="'$(IncludeReleaseNotes)' == 'true'">
<None Include="../../releaseNotes/v$(PackageVersion).md" Pack="true" PackagePath="/" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="../NetInteractor/NetInteractor.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Playwright" />
</ItemGroup>
</Project>
181 changes: 181 additions & 0 deletions src/NetInteractor.Playwright/PlaywrightWebAccessor.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,181 @@
using System;
using System.Collections.Specialized;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Threading;
using System.Threading.Tasks;
using System.Text;
using Microsoft.Playwright;
using NetInteractor.Config;

namespace NetInteractor.WebAccessors
{
public class PlaywrightWebAccessor : IWebAccessor, IAsyncDisposable
{
private IPlaywright _playwright;
private IBrowser _browser;
private readonly SemaphoreSlim _browserLock = new SemaphoreSlim(1, 1);
private readonly BrowserTypeLaunchOptions _launchOptions;
private bool _disposed;

public PlaywrightWebAccessor(BrowserTypeLaunchOptions launchOptions = null)
{
_launchOptions = launchOptions ?? new BrowserTypeLaunchOptions
{
Headless = true,
Args = new[] { "--no-sandbox", "--disable-setuid-sandbox" }
};
}

private async Task<IBrowser> GetBrowserAsync()
{
if (_browser == null)
{
await _browserLock.WaitAsync();
try
{
if (_browser == null)
{
_playwright = await Microsoft.Playwright.Playwright.CreateAsync();
_browser = await _playwright.Chromium.LaunchAsync(_launchOptions);
}
}
finally
{
_browserLock.Release();
}
}
return _browser;
}

public async Task<ResponseInfo> GetAsync(string url, InteractActionConfig config = null)
{
var browser = await GetBrowserAsync();
var page = await browser.NewPageAsync();

try
{
var response = await page.GotoAsync(url, new PageGotoOptions
{
WaitUntil = WaitUntilState.NetworkIdle
});

var loadDelayStr = config?.Options?.FirstOrDefault(attr => attr.Name == "loadDelay")?.Value;
if (!string.IsNullOrEmpty(loadDelayStr) && int.TryParse(loadDelayStr, out var loadDelay))
{
var delayTask = page.WaitForTimeoutAsync(loadDelay);
var navigationTask = page.WaitForNavigationAsync(new PageWaitForNavigationOptions

Check warning on line 69 in src/NetInteractor.Playwright/PlaywrightWebAccessor.cs

View workflow job for this annotation

GitHub Actions / build (ubuntu-latest)

'IPage.WaitForNavigationAsync(PageWaitForNavigationOptions?)' is obsolete

Check warning on line 69 in src/NetInteractor.Playwright/PlaywrightWebAccessor.cs

View workflow job for this annotation

GitHub Actions / build (ubuntu-latest)

'IPage.WaitForNavigationAsync(PageWaitForNavigationOptions?)' is obsolete

Check warning on line 69 in src/NetInteractor.Playwright/PlaywrightWebAccessor.cs

View workflow job for this annotation

GitHub Actions / build (ubuntu-latest)

'IPage.WaitForNavigationAsync(PageWaitForNavigationOptions?)' is obsolete

Check warning on line 69 in src/NetInteractor.Playwright/PlaywrightWebAccessor.cs

View workflow job for this annotation

GitHub Actions / build (ubuntu-latest)

'IPage.WaitForNavigationAsync(PageWaitForNavigationOptions?)' is obsolete

Check warning on line 69 in src/NetInteractor.Playwright/PlaywrightWebAccessor.cs

View workflow job for this annotation

GitHub Actions / build (ubuntu-latest)

'IPage.WaitForNavigationAsync(PageWaitForNavigationOptions?)' is obsolete

Check warning on line 69 in src/NetInteractor.Playwright/PlaywrightWebAccessor.cs

View workflow job for this annotation

GitHub Actions / build (ubuntu-latest)

'IPage.WaitForNavigationAsync(PageWaitForNavigationOptions?)' is obsolete

Check warning on line 69 in src/NetInteractor.Playwright/PlaywrightWebAccessor.cs

View workflow job for this annotation

GitHub Actions / build (ubuntu-latest)

'IPage.WaitForNavigationAsync(PageWaitForNavigationOptions?)' is obsolete

Check warning on line 69 in src/NetInteractor.Playwright/PlaywrightWebAccessor.cs

View workflow job for this annotation

GitHub Actions / build (macos-latest)

'IPage.WaitForNavigationAsync(PageWaitForNavigationOptions?)' is obsolete

Check warning on line 69 in src/NetInteractor.Playwright/PlaywrightWebAccessor.cs

View workflow job for this annotation

GitHub Actions / build (macos-latest)

'IPage.WaitForNavigationAsync(PageWaitForNavigationOptions?)' is obsolete

Check warning on line 69 in src/NetInteractor.Playwright/PlaywrightWebAccessor.cs

View workflow job for this annotation

GitHub Actions / build (macos-latest)

'IPage.WaitForNavigationAsync(PageWaitForNavigationOptions?)' is obsolete

Check warning on line 69 in src/NetInteractor.Playwright/PlaywrightWebAccessor.cs

View workflow job for this annotation

GitHub Actions / build (macos-latest)

'IPage.WaitForNavigationAsync(PageWaitForNavigationOptions?)' is obsolete

Check warning on line 69 in src/NetInteractor.Playwright/PlaywrightWebAccessor.cs

View workflow job for this annotation

GitHub Actions / build (macos-latest)

'IPage.WaitForNavigationAsync(PageWaitForNavigationOptions?)' is obsolete

Check warning on line 69 in src/NetInteractor.Playwright/PlaywrightWebAccessor.cs

View workflow job for this annotation

GitHub Actions / build (macos-latest)

'IPage.WaitForNavigationAsync(PageWaitForNavigationOptions?)' is obsolete

Check warning on line 69 in src/NetInteractor.Playwright/PlaywrightWebAccessor.cs

View workflow job for this annotation

GitHub Actions / build (macos-latest)

'IPage.WaitForNavigationAsync(PageWaitForNavigationOptions?)' is obsolete

Check warning on line 69 in src/NetInteractor.Playwright/PlaywrightWebAccessor.cs

View workflow job for this annotation

GitHub Actions / build (windows-latest)

'IPage.WaitForNavigationAsync(PageWaitForNavigationOptions?)' is obsolete

Check warning on line 69 in src/NetInteractor.Playwright/PlaywrightWebAccessor.cs

View workflow job for this annotation

GitHub Actions / build (windows-latest)

'IPage.WaitForNavigationAsync(PageWaitForNavigationOptions?)' is obsolete

Check warning on line 69 in src/NetInteractor.Playwright/PlaywrightWebAccessor.cs

View workflow job for this annotation

GitHub Actions / build (windows-latest)

'IPage.WaitForNavigationAsync(PageWaitForNavigationOptions?)' is obsolete

Check warning on line 69 in src/NetInteractor.Playwright/PlaywrightWebAccessor.cs

View workflow job for this annotation

GitHub Actions / build (windows-latest)

'IPage.WaitForNavigationAsync(PageWaitForNavigationOptions?)' is obsolete

Check warning on line 69 in src/NetInteractor.Playwright/PlaywrightWebAccessor.cs

View workflow job for this annotation

GitHub Actions / build (windows-latest)

'IPage.WaitForNavigationAsync(PageWaitForNavigationOptions?)' is obsolete

Check warning on line 69 in src/NetInteractor.Playwright/PlaywrightWebAccessor.cs

View workflow job for this annotation

GitHub Actions / build (windows-latest)

'IPage.WaitForNavigationAsync(PageWaitForNavigationOptions?)' is obsolete

Check warning on line 69 in src/NetInteractor.Playwright/PlaywrightWebAccessor.cs

View workflow job for this annotation

GitHub Actions / build (windows-latest)

'IPage.WaitForNavigationAsync(PageWaitForNavigationOptions?)' is obsolete
{
WaitUntil = WaitUntilState.NetworkIdle
});

var completedTask = await Task.WhenAny(delayTask, navigationTask);
if (completedTask == navigationTask)
{
try
{
response = await navigationTask;
}
catch (PlaywrightException)
{
}
}
}

return await GetResultFromResponse(page, response);
}
finally
{
await page.CloseAsync();
}
}

public async Task<ResponseInfo> PostAsync(string url, NameValueCollection formValues, InteractActionConfig config = null)
{
var browser = await GetBrowserAsync();
var page = await browser.NewPageAsync();

try
{
var formData = string.Join("&", formValues.Keys.OfType<string>()
.Select(k => k + "=" + Uri.EscapeDataString(formValues[k] ?? string.Empty)));

await page.RouteAsync("**/*", async route =>
{
var request = route.Request;
if (request.Url == url && request.Method == "GET")
{
await route.ContinueAsync(new RouteContinueOptions
{
Method = "POST",
PostData = Encoding.UTF8.GetBytes(formData),
Headers = request.Headers.Concat(new[]
{
new KeyValuePair<string, string>("Content-Type", "application/x-www-form-urlencoded")
}).ToDictionary(x => x.Key, x => x.Value)
});
}
else
{
await route.ContinueAsync();
}
});

var response = await page.GotoAsync(url, new PageGotoOptions
{
WaitUntil = WaitUntilState.NetworkIdle
});

return await GetResultFromResponse(page, response);
}
finally
{
await page.CloseAsync();
}
}

private async Task<ResponseInfo> GetResultFromResponse(IPage page, IResponse response)
{
var html = await page.ContentAsync();

var mockResponse = new HttpResponseMessage();
IEnumerable<KeyValuePair<string, string>> headerPairs = response?.Headers ?? Enumerable.Empty<KeyValuePair<string, string>>();

foreach (var header in headerPairs)
{
try
{
mockResponse.Headers.TryAddWithoutValidation(header.Key, header.Value);
}
catch (Exception)
{
}
}

return new ResponseInfo
{
StatusCode = (int)(response?.Status ?? 0),
StatusDescription = response?.StatusText,
Html = html,
Url = response?.Url,
Headers = mockResponse.Headers
};
}

public async ValueTask DisposeAsync()
{
if (!_disposed)
{
_disposed = true;
if (_browser != null)
{
await _browser.DisposeAsync();
}
_playwright?.Dispose();
_browserLock.Dispose();
}
}
}
}
7 changes: 7 additions & 0 deletions test/NetInteractor.Test/IntegrationTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,13 @@ public IEnumerator<object[]> GetEnumerator()
{
yield return new object[] { new PuppeteerSharpWebAccessor(), _kestrelFactory.ServerUrl };
}

// Playwright accessor with Kestrel (real HTTP) - disabled by default in CI/CD
// Requires browser download; enable locally via ENABLE_PLAYWRIGHT_TESTS=true
if (Environment.GetEnvironmentVariable("ENABLE_PLAYWRIGHT_TESTS") == "true")
{
yield return new object[] { new PlaywrightWebAccessor(), _kestrelFactory.ServerUrl };
}
}

IEnumerator IEnumerable.GetEnumerator() => GetEnumerator();
Expand Down
1 change: 1 addition & 0 deletions test/NetInteractor.Test/NetInteractor.Test.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
<ItemGroup>
<ProjectReference Include="../../src/NetInteractor/NetInteractor.csproj" />
<ProjectReference Include="../../src/NetInteractor.PuppeteerSharp/NetInteractor.PuppeteerSharp.csproj" />
<ProjectReference Include="../../src/NetInteractor.Playwright/NetInteractor.Playwright.csproj" />
</ItemGroup>
<ItemGroup>
<None Include="Html\*.html" CopyToOutputDirectory="Always" />
Expand Down