Tests: Split Playwrights tests (#7107)
What changed, and why it matters
This commit reorganizes automated browser tests by splitting some Playwright tests into a second CI job. It does not change any production code, user-facing behavior, or security-sensitive logic. The only non-test code change is a small tolerance adjustment in a refund amount assertion, which is still a test-only change.
No security action needed. This is a test infrastructure and CI configuration change only.
Security signals we found
No strong security signals were identified.
Evidence from the diff
The commit splits Playwright tests into two CI groups (‘Playwright’ and ‘Playwright-2’) by changing xUnit traits on test methods and adding a new CircleCI job. It also removes duplicate Playwright traits from two tests in PlaywrightTests.cs that already have a Lightning trait. A minor change in SubscriptionTests.cs widens an assertion tolerance from 3.0m to 4.0m for refund amount verification. No application code is modified.
Changed components
BTCPayServer.Tests/ApiKeysTests.csBTCPayServer.Tests/EmailsTests.csBTCPayServer.Tests/MonetizationTests.csBTCPayServer.Tests/POSTests.csBTCPayServer.Tests/PlaywrightTests.csBTCPayServer.Tests/SubscriptionTests.csBTCPayServer.Tests/WalletTests.cs.circleci/config.ymlInspect captured patch +37 / −26
diff --git a/.circleci/config.yml b/.circleci/config.yml
index 0db0e5c..7853f8d 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -22,6 +22,20 @@ jobs:
docker run --rm -v btcpayservertests_tests_datadir:/data -v /tmp/Artifacts:/host alpine sh -c "cp -r /data/. /host/"
- store_artifacts:
path: /tmp/Artifacts
+ playwright_2_tests:
+ machine:
+ image: ubuntu-2004:2024.11.1
+ steps:
+ - checkout
+ - run:
+ command: |
+ cd .circleci && ./run-tests.sh "Playwright=Playwright-2"
+ - run:
+ when: always
+ command: |
+ docker run --rm -v btcpayservertests_tests_datadir:/data -v /tmp/Artifacts:/host alpine sh -c "cp -r /data/. /host/"
+ - store_artifacts:
+ path: /tmp/Artifacts
selenium_tests:
machine:
image: ubuntu-2004:2024.11.1
@@ -81,6 +95,7 @@ workflows:
jobs:
- fast_tests
- playwright_tests
+ - playwright_2_tests
- selenium_tests
- integration_tests
publish:
diff --git a/BTCPayServer.Tests/ApiKeysTests.cs b/BTCPayServer.Tests/ApiKeysTests.cs
index edffc79..6b345a7 100644
--- a/BTCPayServer.Tests/ApiKeysTests.cs
+++ b/BTCPayServer.Tests/ApiKeysTests.cs
@@ -12,10 +12,8 @@ using BTCPayServer.Views.Manage;
using Microsoft.Playwright;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
-
using Xunit;
using Xunit.Abstractions;
-using StoreData = BTCPayServer.Data.StoreData;
namespace BTCPayServer.Tests
{
@@ -29,7 +27,7 @@ namespace BTCPayServer.Tests
}
[Fact(Timeout = TestTimeout)]
- [Trait("Playwright", "Playwright")]
+ [Trait("Playwright", "Playwright-2")]
public async Task CanCreateApiKeys()
{
//there are 2 ways to create api keys:
diff --git a/BTCPayServer.Tests/EmailsTests.cs b/BTCPayServer.Tests/EmailsTests.cs
index 0810a13..65ee9df 100644
--- a/BTCPayServer.Tests/EmailsTests.cs
+++ b/BTCPayServer.Tests/EmailsTests.cs
@@ -214,7 +214,7 @@ public class EmailsTests(ITestOutputHelper helper) : UnitTestBase(helper)
}
[Fact]
- [Trait("Playwright", "Playwright")]
+ [Trait("Playwright", "Playwright-2")]
public async Task CanSetupEmailRules()
{
await using var s = CreatePlaywrightTester(newDb: true);
diff --git a/BTCPayServer.Tests/MonetizationTests.cs b/BTCPayServer.Tests/MonetizationTests.cs
index 3c6fd46..d2a37ea 100644
--- a/BTCPayServer.Tests/MonetizationTests.cs
+++ b/BTCPayServer.Tests/MonetizationTests.cs
@@ -17,7 +17,7 @@ namespace BTCPayServer.Tests;
public class MonetizationTests(ITestOutputHelper helper) : UnitTestBase(helper)
{
[Fact]
- [Trait("Playwright", "Playwright")]
+ [Trait("Playwright", "Playwright-2")]
public async Task CanMonetizeServer()
{
await using var s = CreatePlaywrightTester(newDb: true);
diff --git a/BTCPayServer.Tests/POSTests.cs b/BTCPayServer.Tests/POSTests.cs
index 314b431..a1daecf 100644
--- a/BTCPayServer.Tests/POSTests.cs
+++ b/BTCPayServer.Tests/POSTests.cs
@@ -737,7 +737,7 @@ goodies:
=> new InvoiceCheckoutPMO(s).AssertContent(new() { AmountDue = expectedAmount });
[Fact]
- [Trait("Playwright", "Playwright")]
+ [Trait("Playwright", "Playwright-2")]
public async Task CanUsePOSKeypad()
{
await using var s = CreatePlaywrightTester();
@@ -1045,7 +1045,7 @@ goodies:
await s.CreateNewStore();
await s.GenerateWallet();
(_, string appId) = await s.CreateApp("PointOfSale");
-
+
await s.Page.Locator("#Title").ClearAsync();
await s.Page.FillAsync("#Title", "Tea shop");
await s.Page.ClickAsync("label[for='DefaultView_Cart']");
@@ -1057,7 +1057,7 @@ goodies:
await s.Page.Locator("#CodeTabButton").WaitForAsync();
await s.Page.Locator("#CodeTabButton").ScrollIntoViewIfNeededAsync();
await s.Page.ClickAsync("#CodeTabButton");
-
+
// Wait for the textarea to be populated by Vue.js
await s.Page.Locator("#TemplateConfig").WaitForAsync();
var template = await s.Page.Locator("#TemplateConfig").InputValueAsync();
@@ -1111,7 +1111,7 @@ goodies:
await s.GoToHome();
await s.GoToServer(ServerNavPages.Policies);
await s.Page.Locator("#RootAppId").ScrollIntoViewIfNeededAsync();
-
+
var options = await s.Page.Locator("#RootAppId option").AllTextContentsAsync();
var targetOption = options.FirstOrDefault(o => o.Contains("Point of"));
if (targetOption != null)
@@ -1124,7 +1124,7 @@ goodies:
}
await s.ClickPagePrimary();
await s.FindAlertMessage();
-
+
// Make sure after login, we are not redirected to the PoS
await s.Logout();
await s.LogIn(userId);
@@ -1132,13 +1132,13 @@ goodies:
content = await s.Page.ContentAsync();
Assert.DoesNotContain("Tea shop", content);
var prevUrl = s.Page.Url;
-
+
// We are only if explicitly going to /
await s.GoToUrl("/");
await s.Page.WaitForLoadStateAsync();
content = await s.Page.ContentAsync();
Assert.Contains("Tea shop", content);
-
+
// Check redirect to canonical url
await s.GoToUrl(posBaseUrl);
Assert.Equal("/", new Uri(s.Page.Url, UriKind.Absolute).AbsolutePath);
@@ -1152,7 +1152,7 @@ goodies:
await s.Page.Locator("#RootAppId").ScrollIntoViewIfNeededAsync();
await s.Page.ClickAsync("#AddDomainButton");
await s.Page.Locator("#DomainToAppMapping_0__Domain").FillAsync(new Uri(s.Page.Url, UriKind.Absolute).DnsSafeHost);
-
+
var domainOptions = await s.Page.Locator("#DomainToAppMapping_0__AppId option").AllTextContentsAsync();
var targetDomainOption = domainOptions.FirstOrDefault(o => o.Contains("Point of"));
if (targetDomainOption != null)
@@ -1165,20 +1165,20 @@ goodies:
}
await s.ClickPagePrimary();
await s.FindAlertMessage(partialText: "Policies updated successfully");
-
+
// Make sure after login, we are not redirected to the PoS
await s.Logout();
await s.LogIn(userId);
await s.Page.WaitForLoadStateAsync();
content = await s.Page.ContentAsync();
Assert.DoesNotContain("Tea shop", content);
-
+
// We are only if explicitly going to /
await s.GoToUrl("/");
await s.Page.WaitForLoadStateAsync();
content = await s.Page.ContentAsync();
Assert.Contains("Tea shop", content);
-
+
// Check redirect to canonical url
await s.GoToUrl(posBaseUrl);
Assert.Equal("/", new Uri(s.Page.Url, UriKind.Absolute).AbsolutePath);
@@ -1190,20 +1190,20 @@ goodies:
await s.GoToUrl($"/apps/{appId}/settings/pos");
}
Assert.Equal(0, await s.Page.Locator("text='Archived App'").CountAsync());
-
+
await s.Page.Locator("#btn-archive-toggle").WaitForAsync();
await s.Page.Locator("#btn-archive-toggle").ScrollIntoViewIfNeededAsync();
await s.Page.ClickAsync("#btn-archive-toggle");
await s.FindAlertMessage(partialText: "The app has been archived and will no longer appear in the apps list by default.");
Assert.Equal(0, await s.Page.Locator("#ViewApp").CountAsync());
-
+
await s.GoToStore(s.StoreId);
var archivedLink = s.Page.Locator("text='1 Archived App'");
await archivedLink.WaitForAsync();
var archivedText = await archivedLink.TextContentAsync();
Assert.Contains("1 Archived App", archivedText);
-
+
await s.GoToUrl(posBaseUrl);
var title = await s.Page.TitleAsync();
Assert.Contains("Page not found", title, StringComparison.OrdinalIgnoreCase);
diff --git a/BTCPayServer.Tests/PlaywrightTests.cs b/BTCPayServer.Tests/PlaywrightTests.cs
index 0ad21b3..f041a34 100644
--- a/BTCPayServer.Tests/PlaywrightTests.cs
+++ b/BTCPayServer.Tests/PlaywrightTests.cs
@@ -2147,7 +2147,6 @@ namespace BTCPayServer.Tests
}
[Fact]
- [Trait("Playwright", "Playwright")]
[Trait("Lightning", "Lightning")]
public async Task CanUsePredefinedRoles()
{
@@ -2584,7 +2583,6 @@ namespace BTCPayServer.Tests
}
[Fact]
- [Trait("Playwright", "Playwright")]
[Trait("Lightning", "Lightning")]
public async Task CanAccessUserStoreAsAdmin()
{
diff --git a/BTCPayServer.Tests/SubscriptionTests.cs b/BTCPayServer.Tests/SubscriptionTests.cs
index 283335f..3da03d8 100644
--- a/BTCPayServer.Tests/SubscriptionTests.cs
+++ b/BTCPayServer.Tests/SubscriptionTests.cs
@@ -55,7 +55,7 @@ public class SubscriptionTests(ITestOutputHelper testOutputHelper) : UnitTestBas
}
[Fact]
- [Trait("Playwright", "Playwright")]
+ [Trait("Playwright", "Playwright-2")]
public async Task CanEditOfferingAndPlans()
{
await using var s = CreatePlaywrightTester();
@@ -185,7 +185,7 @@ public class SubscriptionTests(ITestOutputHelper testOutputHelper) : UnitTestBas
}
[Fact]
- [Trait("Playwright", "Playwright")]
+ [Trait("Playwright", "Playwright-2")]
public async Task CanUpgradeAndDowngrade()
{
await using var s = CreatePlaywrightTester();
@@ -571,7 +571,7 @@ public class SubscriptionTests(ITestOutputHelper testOutputHelper) : UnitTestBas
}
[Fact]
- [Trait("Playwright", "Playwright")]
+ [Trait("Playwright", "Playwright-2")]
public async Task CanCreateSubscriberAndCircleThroughStates()
{
await using var s = CreatePlaywrightTester();
@@ -1033,7 +1033,7 @@ public class SubscriptionTests(ITestOutputHelper testOutputHelper) : UnitTestBas
var match = Regex.Match(text!, @"\((.*?) USD has been refunded\)");
var v = decimal.Parse(match.Groups[1].Value, CultureInfo.InvariantCulture);
var diff = Math.Abs(refunded - v);
- if (diff >= 3.0m)
+ if (diff >= 4.0m)
{
Assert.Fail($"Expected {refunded} USD, but got {v} USD");
}
diff --git a/BTCPayServer.Tests/WalletTests.cs b/BTCPayServer.Tests/WalletTests.cs
index 4e7dcfd..a78b9ba 100644
--- a/BTCPayServer.Tests/WalletTests.cs
+++ b/BTCPayServer.Tests/WalletTests.cs
@@ -116,7 +116,7 @@ public class WalletTests(ITestOutputHelper helper) : UnitTestBase(helper)
[Fact]
- [Trait("Playwright", "Playwright")]
+ [Trait("Playwright", "Playwright-2")]
public async Task CanUseCPFP()
{
await using var s = CreatePlaywrightTester();
Why this scored 15/100
Community notes
Notes can correct, qualify, or add evidence to the AI analysis. Every note shown here has been validated by a human moderator.
The AI analysis stands alone for now. Submit a note if you can add evidence or important context.