diff --git a/.env.production.example b/.env.production.example index f807c91..cee47da 100644 --- a/.env.production.example +++ b/.env.production.example @@ -1,7 +1,7 @@ # Production deployments intentionally have no working secret defaults. # Copy this file to .env.production, replace every CHANGE-ME value, keep the # result out of version control, and pin a released OSApplyTrack version. -OSAPPLYTRACK_VERSION=1.66.6 +OSAPPLYTRACK_VERSION=1.67.0 POSTGRES_USER=applytrack POSTGRES_PASSWORD=CHANGE-ME-use-a-long-random-password POSTGRES_DB=applytrack diff --git a/BACKLOG.md b/BACKLOG.md index a099068..ff42328 100644 --- a/BACKLOG.md +++ b/BACKLOG.md @@ -162,9 +162,33 @@ review items and unsupported forms still need human attention. - [x] [#394 — Due view: follow up agentically — the agent drafts (and optionally sends) the follow-up](https://github.com/CryptoJones/OSApplyTrack/issues/394) (fixed in 1.66.0: **Follow up** on each Due row and **Follow up on all** walk the follow-ups in one dialog — the tenant's model (any OpenAI-compatible endpoint, metered like any draft, no call at all with drafting off) writes a short email from the application and its dates, the contact, the cover letter, the notes log and the last messages the mailbox has exchanged with that contact; the person edits it, then **Send** mails it from their **own** account (new SMTP host/port on the mailbox settings, same username and sealed app password, host pinned to a public address like the IMAP side) — never the instance's sender — or **Mark followed up** logs one made elsewhere; either way it lands in the notes log and the date moves or clears. No contact on file: the row says so and the draft is still written, to copy. Unreviewed sending split out as #398) - [x] [#401 — Mobile: Errors and Due tables overflow or crush on a phone — Retry cut off](https://github.com/CryptoJones/OSApplyTrack/issues/401) (fixed in 1.66.1) - [ ] [#398 — Follow-ups: opt-in to let the agent send a drafted follow-up without review](https://github.com/CryptoJones/OSApplyTrack/issues/398) -- [ ] [#408 — Agent: LinkedIn SDUI Easy Apply — the contact step's City typeahead is still left unmapped (SMX)](https://github.com/CryptoJones/OSApplyTrack/issues/408) (found re-running the Ready lane after #406: `CityLabel()` matches and the résumé location is typed, but the typeahead is never committed on the SDUI dialog; the run stops at the contact step with `City` the only unmapped field) +- [x] [#408 — Agent: LinkedIn SDUI Easy Apply — the contact step's City typeahead is still left unmapped (SMX)](https://github.com/CryptoJones/OSApplyTrack/issues/408) (fixed in 1.67.0: the "City" was a Work-experience role editor LinkedIn opens on Next, not the contact City; the driver now answers what Next brings up, names each role's fields, and fills Description from the résumé. Found re-running the Ready lane after #406: `CityLabel()` matches and the résumé location is typed, but the typeahead is never committed on the SDUI dialog; the run stops at the contact step with `City` the only unmapped field) - [ ] [#410 — Agent: a packet's whole answer draft rides one LLM completion under the API's 120 s timeout — Orbis's 29-question form timed out twice and every answer was voided](https://github.com/CryptoJones/OSApplyTrack/issues/410) (not the network or the endpoint — a 36 KB request from the agent's own netns answers in 1.5 s; the real 22-question prompt just takes poolside longer than two minutes. Mitigated on pluto with `Llm__TimeoutSeconds=600` on the agent; the fix is a worker-side ceiling, chunked drafting, or both) -- [ ] [#409 — Agent: LinkedIn SDUI Easy Apply — Trility's dialog never reaches Submit within 12 steps](https://github.com/CryptoJones/OSApplyTrack/issues/409) (`mapped: 0, unmapped: []` — the driver presses Next twelve times on a step it cannot leave and reports nothing about it; needs the screenshot and the live dialog) +- [x] [#409 — Agent: LinkedIn SDUI Easy Apply — Trility's dialog never reaches Submit within 12 steps](https://github.com/CryptoJones/OSApplyTrack/issues/409) (fixed in 1.67.0: the dialog really is 14 pages; the limit is 40 and a Next that changes nothing twice fails naming the page. Was: `mapped: 0, unmapped: []` — the driver presses Next twelve times on a step it cannot leave and reports nothing about it; needs the screenshot and the live dialog) +- [x] [#431 — Agent: a checkbox group is discovered as one Yes/No question per option (Fieldwire, Stripe) — fixed in 1.67.0: one multiselect titled by its legend; old split packets still fill](https://github.com/CryptoJones/OSApplyTrack/issues/431) +- [x] [#432 — Agent: a react-select question is discovered with no options, so drafted prose matches no choice — fixed in 1.67.0: menus are opened and read; prose is fitted to the option it means](https://github.com/CryptoJones/OSApplyTrack/issues/432) +- [x] [#433 — Agent: Greenhouse education picked from the résumé stays listed unmapped (Stripe) — fixed in 1.67.0](https://github.com/CryptoJones/OSApplyTrack/issues/433) +- [x] [#434 — Agent: an employer page with ?ashby_jid= is not followed to Ashby's form (Hercules) — fixed in 1.67.0](https://github.com/CryptoJones/OSApplyTrack/issues/434) +- [x] [#435 — Agent: a posting that renders its gone-notice late is reported as an unfillable form (Torentify, Fullstack) — fixed in 1.67.0: retired as closed](https://github.com/CryptoJones/OSApplyTrack/issues/435) +- [x] [#436 — Agent: page scripts cannot see a form drawn inside a shadow root (Manatal careers-page.com) — fixed in 1.67.0](https://github.com/CryptoJones/OSApplyTrack/issues/436) +- [x] [#437 — Agent: Flexhire's notice blocks the résumé button and its Filestack picker is never finished — fixed in 1.67.0](https://github.com/CryptoJones/OSApplyTrack/issues/437) +- [x] [#438 — Agent: phone numbers rejected — an international widget's dial code, and a résumé parse overwriting the box (Motion Recruitment, Robert Half) — fixed in 1.67.0](https://github.com/CryptoJones/OSApplyTrack/issues/438) +- [x] [#439 — Agent: a generic placeholder ("Enter") is taken for a field's label — fixed in 1.67.0](https://github.com/CryptoJones/OSApplyTrack/issues/439) +- [ ] [#440 — Agent: Manatal (careers-page.com) Radix selects and checkbox are not driven](https://github.com/CryptoJones/OSApplyTrack/issues/440) (Yo AI Labs: after #436 the résumé attaches and the hidden questions are found, but the Nationalities / years / salary Radix selects and the terms checkbox are custom widgets neither discovered nor driven) +- [x] [#417 — Agent: a résumé dropzone whose wrapping label is only "Click or drag & drop PDF" is never recognised as the résumé (Ethos) — fixed in 1.67.0: dropzone text never names a file field; its row's title does](https://github.com/CryptoJones/OSApplyTrack/issues/417) +- [x] [#418 — Agent: an application form folded behind "Easy Apply" is never revealed (N-iX) — fixed in 1.67.0: "Easy/Quick/One-click Apply" open the form (N-iX itself then ends at a reCAPTCHA v2 handoff)](https://github.com/CryptoJones/OSApplyTrack/issues/418) +- [x] [#419 — Agent: a footer newsletter signup (mc4wp) is discovered as an application question (Tecbrains) — fixed in 1.67.0: newsletter forms are page widgets](https://github.com/CryptoJones/OSApplyTrack/issues/419) +- [x] [#420 — Agent: Lever's invisible hCaptcha holds Submit and nothing is posted — fixed in 1.67.0: a Submit that sends nothing on a form with a captcha widget is a captcha handoff; parseResume is uploader chatter](https://github.com/CryptoJones/OSApplyTrack/issues/420) +- [x] [#421 — Agent: Ashby's GraphQL submit answer is never read — fixed in 1.67.0: FormSubmitSuccess confirms, errors[].message is reported](https://github.com/CryptoJones/OSApplyTrack/issues/421) +- [x] [#422 — Agent: drafter: "hourly" matched the website rule's "url" — fixed in 1.67.0: "url" as a whole word; hourly rate is a salary question](https://github.com/CryptoJones/OSApplyTrack/issues/422) +- [x] [#423 — Agent: the Apply finder took English "Candidate …" for Apply (Ford) and missed Spanish "Postular" — fixed in 1.67.0](https://github.com/CryptoJones/OSApplyTrack/issues/423) +- [x] [#424 — Agent: a form whose Submit is a JavaScript anchor ends "no Submit button found" (Tential, JazzHR) — fixed in 1.67.0](https://github.com/CryptoJones/OSApplyTrack/issues/424) +- [x] [#425 — Agent: required-empty sweep ignored a starred label and a valued placeholder option (HubSpot) — fixed in 1.67.0](https://github.com/CryptoJones/OSApplyTrack/issues/425) +- [x] [#426 — Agent: Apply inside an embedded ATS iframe, or a second Apply on the page Apply led to, was never pressed (Voleon, 7seventy) — fixed in 1.67.0, plus noopener tabs no longer hang and ad vignettes are blocked](https://github.com/CryptoJones/OSApplyTrack/issues/426) +- [x] [#427 — Agent: expired / "no longer accepting profiles" / "taken offline" postings read as "no Apply button" — fixed in 1.67.0: reported as closed](https://github.com/CryptoJones/OSApplyTrack/issues/427) +- [x] [#428 — Agent: Cloudflare's block page read as "no Apply button" — fixed in 1.67.0: "the site refused this server"](https://github.com/CryptoJones/OSApplyTrack/issues/428) +- [x] [#429 — Agent: ClearCompany's returning-applicant email sign-in mid-wizard — fixed in 1.67.0: an email-only page is a sign-in, its Next is never pressed](https://github.com/CryptoJones/OSApplyTrack/issues/429) +- [x] [#430 — Agent: LinkedIn Easy Apply timed out behind the "Job search safety reminder" (KRON) — fixed in 1.67.0: "Continue applying" is pressed](https://github.com/CryptoJones/OSApplyTrack/issues/430) - [x] [#416 — Agent: two controls under one label — OneStream's address Country lost to the phone's country picker](https://github.com/CryptoJones/OSApplyTrack/issues/416) (fixed in 1.66.6: when a locator matches several controls, the one whose id or name is the question's own id wins; a react-select reads empty after a choice, so "prefer the empty one" took the phone's picker every time.) - [x] [#414 — Agent: a consent question that mentions the phone number gets answered with the phone number (Prenuvo SMS opt-in)](https://github.com/CryptoJones/OSApplyTrack/issues/414) (fixed in 1.66.6: the drafter's phone and email rules skip a label that asks for consent, opt-in or agreement, or names SMS, text messages, WhatsApp or subscribing; plain "Phone" / "Mobile phone number" still take the number.) - [x] [#412 — Agent: LinkedIn SDUI years-of-experience text boxes refuse prose answers ("5+ years")](https://github.com/CryptoJones/OSApplyTrack/issues/412) (fixed in 1.66.5: the SDUI dialog draws "How many years …?" as a plain 20-character text box, so the #332 whole-number coercion keyed on `type=number` never fired and the drafter's "10+ years — …" was refused after Next. A label that asks "How many years" now gets the whole number its answer states, and "Less than 1 year" reads 0, not 1. Found on the 2026-10-05 Ready sweep: seven LinkedIn rows rebuilt on Claude answered every question and still dry-ran 0 mapped.) diff --git a/api/ApplyTrack.Api.Tests/AnswerDrafterTests.cs b/api/ApplyTrack.Api.Tests/AnswerDrafterTests.cs index de7642f..b2c2307 100644 --- a/api/ApplyTrack.Api.Tests/AnswerDrafterTests.cs +++ b/api/ApplyTrack.Api.Tests/AnswerDrafterTests.cs @@ -186,6 +186,22 @@ public async Task A_pinned_name_beats_the_resume_on_every_form_whatever_the_fiel Assert.Equal("Byte", packet.Answers["last_name"]); } + [Fact] + public void An_hourly_rate_question_is_a_salary_question_not_a_website_box() + { + // Wise Integration's Lever card: "url" matched inside "hourly", and the rate went out + // as the LinkedIn link (#422). + var rate = new PacketQuestion("cards[af82][field0]", "Could you please share your expected hourly rate for this role? ✱", true, PacketQuestion.Textarea, [], PacketQuestion.Custom); + var (answer, _) = AnswerDrafter.Deterministic(rate, Ctx()); + Assert.DoesNotContain("http", answer ?? ""); + // A saved yearly figure is converted to an hourly one. + var yearly = Ctx() with { Settings = new AgentSettings { SalaryExpectation = "150000", SalaryPeriod = "annual", SalaryCurrency = "USD" } }; + var (hourly, why) = AnswerDrafter.Deterministic(rate, yearly); + Assert.True(hourly is not null && System.Text.RegularExpressions.Regex.IsMatch(hourly, @"^\d"), $"{hourly} / {why}"); + var other = new PacketQuestion("urls[Other]", "Other URL", false, PacketQuestion.Text, [], PacketQuestion.Custom); + Assert.NotNull(AnswerDrafter.Deterministic(other, Ctx() with { Resume = new Resume { Links = [new("GitHub", "https://github.com/x")] } }).Answer); + } + [Fact] public void Country_and_city_are_deterministic_and_the_standing_country_wins() { diff --git a/api/ApplyTrack.Api.Tests/BrowserSubmitterTests.FormShapes.cs b/api/ApplyTrack.Api.Tests/BrowserSubmitterTests.FormShapes.cs new file mode 100644 index 0000000..35b55d2 --- /dev/null +++ b/api/ApplyTrack.Api.Tests/BrowserSubmitterTests.FormShapes.cs @@ -0,0 +1,461 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 Aaron K. Clark + +using ApplyTrack.Api.Agent; +using ApplyTrack.Api.Agent.Browser; +using ApplyTrack.Api.Data; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Logging.Abstractions; + +namespace ApplyTrack.Api.Tests; + +/// +/// Form shapes met on the 2026-10-06 Ready rows (#431–#439): Greenhouse's checkbox groups and +/// react-select menus (Fieldwire, Stripe), its Education pickers, a gone-notice drawn late +/// (Torentify), a form inside a shadow root (Manatal), a notice over the form and a Filestack +/// picker (Flexhire), an international phone box and a résumé parse that rewrites fields +/// (Motion Recruitment, Robert Half), and boxes whose placeholder is only "Enter" (Fullstack). +/// +public sealed partial class BrowserSubmitterTests +{ + private static void MapFormShapeFixtures(WebApplication app) + { + app.MapGet("/jobs/gh-shapes", () => Results.Content(GreenhouseShapesHtml, "text/html")); + app.MapGet("/jobs/gh-education", () => Results.Content(GreenhouseEducationHtml, "text/html")); + app.MapGet("/jobs/late-gone", () => Results.Content(LateGoneHtml, "text/html")); + app.MapGet("/jobs/shadow-form", () => Results.Content(ShadowFormHtml, "text/html")); + app.MapGet("/jobs/notice-picker", () => Results.Content(NoticePickerHtml, "text/html")); + app.MapGet("/jobs/phone-widget", () => Results.Content(PhoneWidgetHtml, "text/html")); + app.MapGet("/jobs/enter-placeholders", () => Results.Content(EnterPlaceholdersHtml, "text/html")); + } + + // A react-select stand-in: an whose menu exists only while open and + // filters on what is typed, a hidden required input holding the committed value, and the + // value posted under the question's name. "multi" keeps several (value container --is-multi). + private const string ComboScript = """ + + """; + + private static string Combo(string id, string label, bool multi = false) => $""" +
+ +
+
+
+ +
+ +
+ + +
+ """; + + private static readonly string GreenhouseShapesHtml = $$""" + + {{ComboScript}} +
+ + {{Combo("question_7", "Are you legally authorized to work in the U.S. without any restrictions?")}} + {{Combo("question_8[]", "Do you have experience coding with any of the languages listed below?", multi: true)}} + {{Combo("question_9", "Please select the country where you currently reside.")}} +
+ Do you have professional working experience with cloud infrastructure tools? * +
+
+
+
+ +
+ + + """; + + private static readonly string GreenhouseEducationHtml = $$""" + + {{ComboScript}} +
+ +

Education

+ {{Combo("school--0", "School")}} + {{Combo("degree--0", "Degree")}} + +
+ + + """; + + // Torentify: the posting page renders, then its script asks for the job and draws the + // gone-notice — after the run's first look. Only the site's own search box is on it. + private const string LateGoneHtml = """ + + +

Loading…

+ + + """; + + // Manatal: the whole application lives in a web component's open shadow root; labels carry + // the only required mark, a star. + private const string ShadowFormHtml = """ + +

Software Developer

+
+ + + """; + + // Flexhire: Apply opens the form under a notice ("we couldn't detect your location") whose + // only button is Ok and whose one link is set in its sentence; "Upload Resume/CV" opens a + // picker that makes its own file input and sends nothing until its Upload is pressed. + private const string NoticePickerHtml = """ + +
+ + + +

Resume/CV *

+ + + + +
+
+
+

Apply to Job

+

This job accepts applicants located in Argentina, Bolivia, Brazil . We couldn't detect your location, set it below to see if you are eligible to apply.

+ +
+
+ + + """; + + // Motion Recruitment's react-international-phone: the box opens on "+1 " and resets itself + // when its value loses the dial code — a fill() wipes it. Robert Half's résumé parse: the + // upload rewrites the name and phone the run had typed. + private const string PhoneWidgetHtml = """ + +
+ + + + +
+ + + """; + + // Fullstack's talent portal: every box's placeholder is "Enter"; the field's title is the + // text above it, with its star. + private const string EnterPlaceholdersHtml = """ + +
+
First Name*
+
Phone*
+
Portfolio URL
+ +
+ + """; + + [SkippableFact] + public async Task A_checkbox_group_and_react_select_menus_are_discovered_whole() + { + // Fieldwire / Stripe: each box of a "select all that apply" was its own Yes/No question, + // the group's id labelled by its first option (#431); react-select questions came back as + // free text with no options, and the drafter wrote prose to them (#432). + Skip.IfNot(Available, "Node Playwright is not installed (npm ci)"); + var questions = await Discoverer().DiscoverAsync($"{_fixtureUrl}/jobs/gh-shapes"); + Assert.NotNull(questions); + var cloud = Assert.Single(questions!, q => q.Id == "question_10[]"); + Assert.Equal(PacketQuestion.MultiSelect, cloud.Type); + Assert.Equal(["AWS", "GCP", "Azure"], cloud.Options); + Assert.Equal("Do you have professional working experience with cloud infrastructure tools?", cloud.Label); + Assert.True(cloud.Required); + Assert.DoesNotContain(questions!, q => q.Id is "GCP" or "Azure"); + var auth = Assert.Single(questions!, q => q.Id == "question_7"); + Assert.Equal(PacketQuestion.Select, auth.Type); + Assert.Equal(["Yes", "No"], auth.Options); + var langs = Assert.Single(questions!, q => q.Id == "question_8[]"); + Assert.Equal(PacketQuestion.MultiSelect, langs.Type); + Assert.Equal(["Ruby on Rails", "Rust", "Both", "Neither"], langs.Options); + } + + [SkippableFact] + public async Task A_packet_that_split_a_checkbox_group_and_wrote_prose_to_menus_still_fills_the_form() + { + // The packets already in Ready: per-box Yes/No questions, and drafted prose for menus + // the drafter never saw. "No" boxes must stay unticked; the prose picks what it means. + Skip.IfNot(Available, "Node Playwright is not installed (npm ci)"); + var packet = new AgentPacket + { + ApplicationName = "fieldwire-backend.md", Provider = "greenhouse", + Questions = + [ + new("first_name", "First Name", true, PacketQuestion.Text, [], PacketQuestion.Standard), + new("question_7", "Are you legally authorized to work in the U.S. without any restrictions?", true, PacketQuestion.Text, [], PacketQuestion.Custom), + new("question_8[]", "Do you have experience coding with any of the languages listed below?", true, PacketQuestion.Text, [], PacketQuestion.Custom), + new("question_9", "Please select the country where you currently reside.", true, PacketQuestion.Text, [], PacketQuestion.Custom), + new("question_10[]", "AWS", true, PacketQuestion.Select, ["Yes", "No"], PacketQuestion.Custom), + new("GCP", "GCP", true, PacketQuestion.Select, ["Yes", "No"], PacketQuestion.Custom), + new("Azure", "Azure", true, PacketQuestion.Select, ["Yes", "No"], PacketQuestion.Custom), + ], + Answers = new() + { + ["first_name"] = "Ada", + ["question_7"] = "US Citizen", + ["question_8[]"] = "Yes — C# and Python are my deepest languages, plus Rust (a hexagonal core). No Ruby on Rails experience.", + ["question_9"] = "United States of America", + ["question_10[]"] = "Yes", ["GCP"] = "No", ["Azure"] = "Yes", + }, + }; + var outcome = await Submitter().RunAsync($"{_fixtureUrl}/jobs/gh-shapes", packet, null, dryRun: false); + Assert.True(outcome.Submitted, outcome.Error + " unmapped: " + string.Join(", ", outcome.Unmapped)); + var post = Assert.Single(_posts); + Assert.Equal("Yes", post["question_7"]); + Assert.Equal("Rust", post["question_8[]"]); + Assert.Equal("US", post["question_9"]); + Assert.Equal("1,3", post["question_10[]"]); + } + + [SkippableFact] + public async Task Education_the_resume_filled_is_not_left_listed_unmapped() + { + // Stripe: the packet's "University of Maryland University College" is no option, the + // résumé's school then filled the picker — and the run still listed it unmapped (#433). + Skip.IfNot(Available, "Node Playwright is not installed (npm ci)"); + var packet = new AgentPacket + { + ApplicationName = "stripe-backend.md", Provider = "greenhouse", + Questions = + [ + new("first_name", "First Name", true, PacketQuestion.Text, [], PacketQuestion.Standard), + new("school--0", "School", true, PacketQuestion.Text, [], PacketQuestion.Custom), + new("degree--0", "Degree", true, PacketQuestion.Text, [], PacketQuestion.Custom), + ], + Answers = new() + { + ["first_name"] = "Ada", + ["school--0"] = "University of Maryland University College", + ["degree--0"] = "Bachelor of Science, Computer and Information Science", + }, + Resume = new Resume { Education = [new("University of Maryland University College", "Bachelor of Science", "Computer and Information Science", "2010 - 2014")] }, + }; + var outcome = await Submitter().RunAsync($"{_fixtureUrl}/jobs/gh-education", packet, null, dryRun: true); + Assert.True(outcome.Filled, outcome.Error); + Assert.Empty(outcome.Unmapped); + Assert.Contains("school--0", outcome.Mapped); + Assert.Contains("degree--0", outcome.Mapped); + } + + [SkippableFact] + public async Task A_gone_notice_drawn_after_the_first_look_retires_the_posting() + { + // Torentify: "Job Not Found" arrived after the closed-posting check, and the run said + // "nothing on this form could be filled" about the site's search box (#435). + Skip.IfNot(Available, "Node Playwright is not installed (npm ci)"); + var outcome = await Submitter().RunAsync($"{_fixtureUrl}/jobs/late-gone", StandardPacket(), (Pdf, "resume.pdf"), dryRun: true); + Assert.True(outcome.Closed, outcome.Error); + Assert.Empty(_posts); + } + + [Fact] + public void Gone_notices_and_ashby_embeds_are_read_from_their_words() + { + Assert.True(BrowserSubmitter.IsClosedPosting("Oops! We couldn’t find that page. The page might have been relocated.")); // Fullstack (#435) + Assert.True(BrowserSubmitter.IsClosedPosting("Job Not Found The job you are looking for could not be found.")); + Assert.False(BrowserSubmitter.IsClosedPosting("We couldn't find a better team to join.")); + // Hercules: ?ashby_jid= on the employer's own careers page (#434). + const string link = "https://hercules.app/careers?ashby_jid=9ee9a087-9c1b-40e3-ab15-f28d42802314&utm_source=x"; + Assert.True(AtsProvider.HasAshbyJobId(link)); + Assert.Equal("https://jobs.ashbyhq.com/hercules/9ee9a087-9c1b-40e3-ab15-f28d42802314/application", AtsProvider.AshbyHostedForm(link)); + Assert.Equal("https://jobs.ashbyhq.com/acme-labs/9ee9a087-9c1b-40e3-ab15-f28d42802314/application", + AtsProvider.AshbyHostedForm("https://careers.acme.io/?ashby_jid=9ee9a087-9c1b-40e3-ab15-f28d42802314", + "")); + Assert.Null(AtsProvider.AshbyHostedForm("https://jobs.ashbyhq.com/hercules/9ee9a087-9c1b-40e3-ab15-f28d42802314")); + Assert.Null(AtsProvider.AshbyHostedForm("https://hercules.app/careers")); + } + + [Fact] + public void A_menu_choice_is_what_the_answer_means() + { + // #432: prose drafted to a menu the drafter never saw. + Assert.Equal(["Yes"], BrowserSubmitter.MenuChoices(["Yes", "No"], "Yes. At Ronin 48 I build AI features.", false)); + Assert.Equal(["No"], BrowserSubmitter.MenuChoices(["Yes", "No"], "No, I do not.", false)); + Assert.Equal(["Yes"], BrowserSubmitter.MenuChoices(["Yes", "No"], "US Citizen", false, "Are you legally authorized to work in the U.S.?")); + Assert.Empty(BrowserSubmitter.MenuChoices(["Yes", "No"], "US Citizen", false, "Will you require sponsorship?")); + Assert.Empty(BrowserSubmitter.MenuChoices(["Yes", "No"], "Not without sponsorship", false)); + Assert.Equal(["US"], BrowserSubmitter.MenuChoices(["Australia", "UK", "US", "Other"], "United States of America", false)); + Assert.Equal(["Rust"], BrowserSubmitter.MenuChoices(["Ruby on Rails", "Rust", "Both", "Neither"], + "C# and Python, plus Rust. No Ruby on Rails experience.", true)); + Assert.Equal(["AWS", "Azure"], BrowserSubmitter.MenuChoices(["AWS", "GCP", "Azure"], "AWS, Azure", true)); + Assert.Empty(BrowserSubmitter.MenuChoices(["Seattle", "New York"], "I live in Nebraska.", false)); + } + + [SkippableFact] + public async Task A_form_inside_a_shadow_root_is_discovered_and_its_resume_seen_to_take() + { + // Manatal (yo-ai-labs): discovery found nothing, and the attached résumé was never seen, + // "required fields could not be mapped (std:resume)" (#436). + Skip.IfNot(Available, "Node Playwright is not installed (npm ci)"); + var questions = await Discoverer().DiscoverAsync($"{_fixtureUrl}/jobs/shadow-form"); + Assert.NotNull(questions); + Assert.Contains(questions!, q => q.Id == "full_name" && q.Label == "What is your full name?" && q.Required); + Assert.Contains(questions!, q => q.Id == "position" && q.Required); + + var packet = StandardPacket(); + var outcome = await Submitter().RunAsync($"{_fixtureUrl}/jobs/shadow-form", packet, (Pdf, "resume.pdf"), dryRun: true); + Assert.Contains("std:resume", outcome.Mapped); + Assert.DoesNotContain("std:resume", outcome.Unmapped); + // The starred position box the packet never knew is named, and handed to the drafter. + Assert.Contains("position", outcome.Unmapped); + Assert.Contains(outcome.Discovered ?? [], q => q.Id == "position"); + } + + [SkippableFact] + public async Task A_notice_over_the_form_is_acknowledged_and_a_picker_upload_is_finished() + { + // Flexhire: the Ok-only notice took the résumé button's click, and the Filestack picker + // made a file input and waited for its own Upload — "Resume is required" (#437). + Skip.IfNot(Available, "Node Playwright is not installed (npm ci)"); + var outcome = await Submitter().RunAsync($"{_fixtureUrl}/jobs/notice-picker", StandardPacket(), (Pdf, "resume.pdf"), dryRun: false); + Assert.True(outcome.Submitted, outcome.Error + " unmapped: " + string.Join(", ", outcome.Unmapped)); + var post = Assert.Single(_posts); + Assert.Equal("resume.pdf", post["resume_name"]); + Assert.Equal("Ada", post["first_name"]); + } + + [SkippableFact] + public async Task A_phone_box_holding_a_dial_code_is_typed_after_it_and_a_resume_parse_does_not_win() + { + // Motion Recruitment: fill() into "+1 " left "+1 " (#438). Robert Half: the résumé went up + // after the typing and its parse rewrote what had been typed (#438). + Skip.IfNot(Available, "Node Playwright is not installed (npm ci)"); + var packet = new AgentPacket + { + ApplicationName = "motion-dotnet.md", Provider = "unknown", + Questions = + [ + new("firstName", "First Name:", true, PacketQuestion.Text, [], PacketQuestion.Standard), + new("phone", "Phone Number:", true, PacketQuestion.Text, [], PacketQuestion.Standard), + new("resume", "Upload Your Resume:", true, PacketQuestion.File, [], PacketQuestion.Standard), + ], + Answers = new() { ["firstName"] = "Ada", ["phone"] = "5712975406" }, + }; + var outcome = await Submitter().RunAsync($"{_fixtureUrl}/jobs/phone-widget", packet, (Pdf, "resume.pdf"), dryRun: false); + Assert.True(outcome.Submitted, outcome.Error); + var post = Assert.Single(_posts); + Assert.Equal("+1 5712975406", post["phone"]); + Assert.Equal("Ada", post["firstName"]); + } + + [Fact] + public void A_dial_code_prefix_and_the_national_number_are_read_apart() + { + Assert.Equal("1", BrowserSubmitter.DialCodePrefix("+1 ")); + Assert.Equal("44", BrowserSubmitter.DialCodePrefix(" +44")); + Assert.Null(BrowserSubmitter.DialCodePrefix("")); + Assert.Null(BrowserSubmitter.DialCodePrefix("+1 (571) 297-5406")); + Assert.Equal("5712975406", BrowserSubmitter.NationalDigits("5712975406", "1")); + Assert.Equal("5712975406", BrowserSubmitter.NationalDigits("+1 571 297 5406", "1")); + Assert.Equal("5712975406", BrowserSubmitter.NationalDigits("15712975406", "1")); + Assert.Equal("1234567890", BrowserSubmitter.NationalDigits("1234567890", "1")); + } + + [SkippableFact] + public async Task A_placeholder_that_only_says_enter_does_not_name_the_field() + { + // Fullstack: every question came back "Enter", optional (#439). + Skip.IfNot(Available, "Node Playwright is not installed (npm ci)"); + var questions = await Discoverer().DiscoverAsync($"{_fixtureUrl}/jobs/enter-placeholders"); + Assert.NotNull(questions); + var phone = Assert.Single(questions!, q => q.Id == "talent.phone"); + Assert.Equal("Phone", phone.Label); + Assert.True(phone.Required); + var portfolio = Assert.Single(questions!, q => q.Id == "talent.portfolio"); + Assert.Equal("Portfolio URL", portfolio.Label); + Assert.False(portfolio.Required); + } +} diff --git a/api/ApplyTrack.Api.Tests/BrowserSubmitterTests.cs b/api/ApplyTrack.Api.Tests/BrowserSubmitterTests.cs index 76c0447..5b90d2c 100644 --- a/api/ApplyTrack.Api.Tests/BrowserSubmitterTests.cs +++ b/api/ApplyTrack.Api.Tests/BrowserSubmitterTests.cs @@ -24,7 +24,7 @@ namespace ApplyTrack.Api.Tests; /// here resolves to loopback and the fixture asserts it, so a stray real URL /// fails loudly instead of applying to someone's job. /// -public sealed class BrowserSubmitterTests : IAsyncLifetime +public sealed partial class BrowserSubmitterTests : IAsyncLifetime { private static readonly string RepoRoot = FindRepoRoot(); private static readonly string Cli = Path.Combine(RepoRoot, "node_modules", "playwright-core", "cli.js"); @@ -81,6 +81,20 @@ public async Task InitializeAsync() """""", """Continue""") .Replace("document.getElementById('entry').onclick = () => {", "document.getElementById('entry').onclick = e => { e.preventDefault();"), "text/html")); + // LinkedIn's "Job search safety reminder" in front of the dialog (KRON Development, 2026-10-05, #430). + _fixture.MapGet("/li/jobs/view/sdui-safety", () => Results.Content(EasyApplySduiHtml + .Replace(" + """, "text/html")); + // N-iX (#418): the whole form folded in a sidebar until "Easy Apply" is pressed — its file + // input in the page, hidden, from the start. + _fixture.MapGet("/jobs/easy-apply-fold", () => Results.Content( + """ +

Lead .Net Engineer

About the role.

+ + + + """, "text/html")); + // Tecbrains (#419): the application form, and a Mailchimp for WordPress signup in the footer. + _fixture.MapGet("/jobs/newsletter-footer", () => Results.Content(FormHtml.Replace("", + """ + + """), "text/html")); + // Lever (#420): Submit only asks an invisible hCaptcha for a token, which never comes, so + // nothing is ever posted. + _fixture.MapGet("/jobs/hcaptcha-held", () => Results.Content( + """ +
+ +
+ + +
+ """, "text/html")); + // Ashby (#421): the application goes up as a GraphQL mutation answered 200 either way, and + // the page here says nothing — the body is the verdict. + foreach (var path in new[] { "graphql-accepted", "graphql-refused" }) + { + _fixture.MapGet($"/jobs/{path}", () => Results.Content( + $$""" + + + + + """, "text/html")); + } + _fixture.MapPost("/api/non-user-graphql", (HttpRequest req) => + { + lock (_posts) _posts.Add(new() { ["graphql"] = req.Query["op"].ToString() }); + return Results.Content(req.Query["r"] == "graphql-accepted" + ? """{"data":{"submitSingleApplicationFormAction":{"applicationFormResult":{"__typename":"FormSubmitSuccess","_":null},"__typename":"SubmitSingleApplicationFormActionResult"}}}""" + : """{"errors":[{"message":"Missing required field: Location","extensions":{}}],"data":null}""", "application/json"); + }); + // Tential / JazzHR: the form's Submit is a script's anchor, not a button (#424). + _fixture.MapGet("/jobs/anchor-submit", () => Results.Content( + """ +

.NET Developer

+ APPLY NOW +
+ +

Privacy Policy

+
APPLY NOW
+
+ + """, "text/html")); + // HubSpot's rendering of its Greenhouse form: required only by the star in the label, a + // placeholder option that carries a value, and Submit disabled until all are answered (#425). + _fixture.MapGet("/jobs/starred-selects", () => Results.Content( + """ +

Principal Engineer

+
+ + + + + + + + +
+ + + """, "text/html")); + // Voleon: the posting (and its Apply) is an embedded ATS page in an iframe (#426). + _fixture.MapGet("/jobs/embedded-posting", () => Results.Content( + """

Voleon - Job Listings

""", "text/html")); + _fixture.MapGet("/jobs/embedded-posting-inner", () => Results.Content( + """

Senior Software Engineer

About the role.

Apply for this Job""", "text/html")); + // 7seventy: an aggregator whose Apply opens the employer's posting in a new tab (target=_blank, + // rel=noopener), and that posting has its own Apply (#426). + _fixture.MapGet("/jobs/aggregator", () => Results.Content( + """

Staff Backend Engineer

Remote.

Apply for This Job""", "text/html")); + _fixture.MapGet("/jobs/ats-posting", () => Results.Content( + """

Staff Backend Engineer @ Astra

Overview.

""", "text/html")); + // Ford: a "Candidate FEEDBACK" tab before the real Apply Now; pressing it opens a survey (#423). + _fixture.MapGet("/jobs/candidate-tab", () => Results.Content( + """ + + + +

Enterprise AI Administrator

+ Apply Now + + """, "text/html")); + // Cloudflare's block page: HTTP 403 and more than 600 characters of explanation (#428). + _fixture.MapGet("/jobs/cloudflare-block", () => Results.Content( + "Attention Required! | Cloudflare

Sorry, you have been blocked

You are unable to access example.com

" + + "

" + string.Join(" ", Enumerable.Repeat("This website is using a security service to protect itself from online attacks.", 10)) + "

", + "text/html", statusCode: 403)); + // ClearCompany's returning applicant: page one of the wizard, Continue, and then an email + // sign-in — a plain text box and NEXT — in place of page two (#429). + _fixture.MapGet("/jobs/returning-applicant", () => Results.Content( + """ +

.NET Software Engineer Application

+ + + +
+ + + """, "text/html")); + // Greenhouse's checkbox groups and react-select menus, Manatal's shadow root, Flexhire's + // notice and picker, an international phone box (#431–#439). + MapFormShapeFixtures(_fixture); _fixture.MapPost("/apply", async (HttpRequest req) => { var form = await req.ReadFormAsync(); @@ -2021,6 +2189,94 @@ function render() { """; + // Copied from the live one (KRON Development, 2026-10-05): a native around the + // PreApplySafetyTipsModal screen whose "Continue applying" is a link back to the posting. + private const string SafetyTipsDialog = """ + + +

Job search safety reminder

+
+

Research the company

Report suspicious jobs

+ +
+ + """; + + private const string LongDialogSteps = """ + steps.splice(3, 0, ...Array.from({ length: 10 }, (_, i) => `

Additional Questions

+

Question number ${i}?*

+
+
`)); + + """; + + // The work-history step as SMX's JobDiva form has it (live, 2026-10-06): the profile's roles as + // cards, "N/A" where the profile holds nothing, and a Next that opens the first short row for + // editing — City* and Description* already flagged, a "Save" of the row's own — instead of + // moving on. The textarea carries the focus key itself, its helper beside it. + private const string WorkHistoryStep = """ + const rows = [ + { title: 'Principal Architect', company: 'Ronin 48', city: 'Omaha, NE', desc: 'Built things.' }, + { title: 'Senior Technical Support Engineer', company: 'CrowdStrike', city: '', desc: '' }, + { title: 'Cloud Operations Supervisor', company: 'CloudHesive', city: '', desc: '' }, + ]; + let editing = -1; + steps.splice(3, 0, 'HISTORY'); + const card = r => `

Your title

${r.title}

Company

${r.company}

+

Dates of employment

8

/

2022

-

7

/

2023

+

City

${r.city || 'N/A'}

Description

${r.desc || 'N/A'}

+
`; + const box = (label, id, value, n) => `

${label}*

+
+
${value ? '' : '

This field is required

'}
`; + const editor = r => `

Edit experience

+ ${box('Your title', 'e-title', r.title, 1)}${box('Company', 'e-company', r.company, 2)} +

Dates of employment

From

+ ${box('City', 'e-city', '', 3)} +

Description*

+

This field is required

+
`; + function historyHtml() { return `

Work experience

${rows.map((r, i) => i === editing ? editor(r) : card(r)).join('')}

You have reached the limit of 10 responses

`; } + function historyNext() { + if (editing >= 0) { + const city = col.querySelector('#e-city').value.trim(), desc = col.querySelector('#e-desc').value.trim(); + if (!city || !desc) return false; + rows[editing].city = city; rows[editing].desc = desc; + } + editing = rows.findIndex(r => !r.city || !r.desc); + if (editing < 0) return true; + col.innerHTML = historyHtml(); + return false; + } + + """; + + private static AgentPacket EasyApplyHistoryPacket(bool citiesKnown) + { + var packet = EasyApplyPacket(answered: true); + packet.Resume = new Resume + { + Location = "Minden, Nebraska (Remote)", + Experience = + [ + new("CrowdStrike", "Senior Technical Support Engineer", "Aug 2022 - Mar 2024", ["SME for LogScale.", "Wrote tooling in Python."]), + new("CloudHesive", "Cloud Operations Supervisor / Cloud Support Engineer II", "Aug 2021 - May 2022", ["Led the Cloud Operations team."]), + ], + }; + if (!citiesKnown) return packet; + foreach (var label in new[] + { + "Work experience — Senior Technical Support Engineer — CrowdStrike — City", + "Work experience — Cloud Operations Supervisor — CloudHesive — City", + }) + { + packet.Questions.Add(new(label, label, true, PacketQuestion.Text, [], PacketQuestion.Custom)); + packet.Answers[label] = "Remote"; + } + return packet; + } + private static AgentPacket EasyApplyPacket(bool answered) { var packet = new AgentPacket { ApplicationName = "acme-senior-.net-engineer.md", Provider = ApplyTrack.Api.Agent.AtsProvider.LinkedInEasy }; @@ -3153,6 +3409,74 @@ public async Task Easy_apply_sdui_dialog_submits_without_following_the_company() lock (_posts) Assert.Single(_posts, p => p.ContainsKey("sdui_submit")); } + [SkippableFact] + public async Task Easy_apply_continues_past_linkedins_safety_reminder() + { + Skip.IfNot(Available, "Node Playwright is not installed (npm ci)"); + // Live 2026-10-05: both KRON postings failed on "Timeout 15000ms exceeded." with the + // "Job search safety reminder" sitting where the dialog should have been (#430). + var outcome = await Submitter().RunAsync($"{_fixtureUrl}/li/jobs/view/sdui-safety", EasyApplyPacket(answered: true), (Pdf, "resume.pdf"), dryRun: true); + + Assert.True(outcome.ReachedSubmit, outcome.Error); + Assert.Empty(outcome.Unmapped); + Assert.Equal("discard", Assert.Single(EasyApplyEvents()).GetProperty("left").GetString()); + } + + [SkippableFact] + public async Task Easy_apply_walks_a_dialog_longer_than_twelve_pages() + { + Skip.IfNot(Available, "Node Playwright is not installed (npm ci)"); + // Trility's dialog is fourteen pages, one question each; every run ended on page 13 with + // "did not reach its Submit step within 12 steps" (#409). + var packet = EasyApplyPacket(answered: true); + for (var i = 0; i < 10; i++) + { + var label = $"Question number {i}?"; + packet.Questions.Add(new(label, label, true, PacketQuestion.Text, [], PacketQuestion.Custom)); + packet.Answers[label] = $"answer {i}"; + } + var outcome = await Submitter().RunAsync($"{_fixtureUrl}/li/jobs/view/sdui-long", packet, (Pdf, "resume.pdf"), dryRun: true); + + Assert.True(outcome.ReachedSubmit, outcome.Error); + Assert.Empty(outcome.Unmapped); + Assert.Equal(14, outcome.Mapped.Count); // the phone, three questions, and ten more + } + + [SkippableFact] + public async Task Easy_apply_hands_back_every_short_work_history_row_and_saves_the_draft() + { + Skip.IfNot(Available, "Node Playwright is not installed (npm ci)"); + // SMX (#408): the "City" every run stopped on was not the person's — it was a past job's, + // in the work-history editor Next opened. Each blank row is that row's question; the + // résumé has each role's lines for its Description but no city for a job gone by. + var outcome = await Submitter().RunAsync($"{_fixtureUrl}/li/jobs/view/sdui-history", EasyApplyHistoryPacket(citiesKnown: false), (Pdf, "resume.pdf"), dryRun: true); + + Assert.False(outcome.Submitted); + Assert.Equal(["Work experience — Senior Technical Support Engineer — CrowdStrike — City"], outcome.Unmapped); + var found = Assert.IsType>(outcome.Discovered).Select(q => q.Label).ToList(); + Assert.Equal( + ["Work experience — Senior Technical Support Engineer — CrowdStrike — City", "Work experience — Cloud Operations Supervisor — CloudHesive — City"], + found); + Assert.Contains("Work experience — Senior Technical Support Engineer — CrowdStrike — Description", outcome.Mapped); + // Saved through the question's own "Save", not the open row's. + var left = Assert.Single(EasyApplyEvents()); + Assert.Equal("save", left.GetProperty("left").GetString()); + Assert.Equal(3, left.GetProperty("step").GetInt32()); + } + + [SkippableFact] + public async Task Easy_apply_fills_each_work_history_row_as_it_opens_and_submits() + { + Skip.IfNot(Available, "Node Playwright is not installed (npm ci)"); + var outcome = await Submitter().RunAsync($"{_fixtureUrl}/li/jobs/view/sdui-history", EasyApplyHistoryPacket(citiesKnown: true), (Pdf, "resume.pdf"), dryRun: false); + + Assert.True(outcome.Submitted, outcome.Error); + var sent = Assert.Single(EasyApplyEvents()).GetProperty("answers"); + Assert.Equal( + "Ronin 48|Omaha, NE|Built things.;CrowdStrike|Remote|SME for LogScale. Wrote tooling in Python.;CloudHesive|Remote|Led the Cloud Operations team.", + sent.GetProperty("history").GetString()); + } + [SkippableFact] public async Task Easy_apply_says_so_when_linkedin_serves_the_signed_out_page() { @@ -3874,7 +4198,8 @@ public async Task A_react_select_combobox_is_driven_to_a_committed_option_and_re Assert.True(outcome.Submitted, outcome.Error); Assert.Empty(outcome.Unmapped); - Assert.Equal(["first_name", "country", "location", "resume", "question_9"], outcome.Mapped); + // Files go first, so a board's résumé parse cannot overwrite the answers (#438). + Assert.Equal(["resume", "first_name", "country", "location", "question_9"], outcome.Mapped); var post = Assert.Single(_posts); Assert.Equal("US", post["country_value"]); Assert.Equal("Omaha, Nebraska, United States", post["location_value"]); @@ -4500,6 +4825,239 @@ public async Task A_posting_answering_410_is_reported_gone_not_formless() Assert.Empty(_posts); } + private FormDiscoverer Discoverer() => + new(new BrowserOptions { Endpoint = _ws, AllowPrivateTargets = true, TimeoutSeconds = 60 }, NullLogger.Instance); + + [SkippableFact] + public async Task A_resume_dropzone_is_named_by_its_field_and_takes_the_resume() + { + Skip.IfNot(Available, "Node Playwright is not installed (npm ci)"); + // Ethos: discovered as "Click or drag & drop PDF", never attached, unmapped on every run (#417). + var questions = await Discoverer().DiscoverAsync($"{_fixtureUrl}/jobs/dropzone"); + var file = Assert.Single(questions!, q => q.Type == PacketQuestion.File); + Assert.Equal("CV / Resume", file.Label); + Assert.True(file.Required); + + // A packet built before the fix still carries the dropzone's words; the one file input is + // the résumé all the same. + var stale = new AgentPacket + { + ApplicationName = "ethos-ai-engineer.md", Provider = "unknown", + Questions = + [ + new("firstName", "First name", true, PacketQuestion.Text, [], PacketQuestion.Standard), + new("email", "Email", true, PacketQuestion.Text, [], PacketQuestion.Standard), + new("Click or drag & drop PDF", "Click or drag & drop PDF", true, PacketQuestion.File, [], PacketQuestion.Custom), + ], + Answers = new() { ["firstName"] = "Ada", ["email"] = "ada@example.com" }, + }; + var outcome = await Submitter().RunAsync($"{_fixtureUrl}/jobs/dropzone", stale, (Pdf, "resume.pdf"), dryRun: false); + Assert.True(outcome.Submitted, outcome.Error); + Assert.Contains("Click or drag & drop PDF", outcome.Mapped); + Assert.Equal("resume.pdf", Assert.Single(_posts)["resume_name"]); + } + + [SkippableFact] + public async Task A_form_folded_behind_easy_apply_is_opened_and_filled() + { + Skip.IfNot(Available, "Node Playwright is not installed (npm ci)"); + // N-iX: "Easy Apply" did not begin with "apply", so only the hidden file input was ever + // seen and first name, last name and email were unmapped (#418). + var outcome = await Submitter().RunAsync($"{_fixtureUrl}/jobs/easy-apply-fold", StandardPacket(), (Pdf, "resume.pdf"), dryRun: false); + Assert.True(outcome.Submitted, outcome.Error); + var post = Assert.Single(_posts); + Assert.Equal("Ada", post["first_name"]); + Assert.Equal("ada@example.com", post["email"]); + } + + [SkippableFact] + public async Task A_newsletter_signup_in_the_footer_is_not_a_question() + { + Skip.IfNot(Available, "Node Playwright is not installed (npm ci)"); + // Tecbrains: the mc4wp box became the question "email", unmapped on every dry run (#419). + var questions = await Discoverer().DiscoverAsync($"{_fixtureUrl}/jobs/newsletter-footer"); + Assert.NotNull(questions); + Assert.Contains(questions!, q => q.Id == "job_application[email]"); + Assert.DoesNotContain(questions!, q => q.Id is "email" or "_mc4wp_honeypot"); + } + + [SkippableFact] + public async Task A_submit_held_by_an_invisible_captcha_is_a_captcha_handoff() + { + Skip.IfNot(Available, "Node Playwright is not installed (npm ci)"); + // Lever: the click asks hCaptcha for a token that never comes; nothing is posted (#420). + var packet = new AgentPacket + { + ApplicationName = "acme-engineer.md", Provider = "lever", + Questions = [new("first_name", "First Name", true, PacketQuestion.Text, [], PacketQuestion.Standard)], + Answers = new() { ["first_name"] = "Ada" }, + }; + var outcome = await Submitter().RunAsync($"{_fixtureUrl}/jobs/hcaptcha-held", packet, null, dryRun: false); + Assert.False(outcome.Submitted); + Assert.True(outcome.Captcha, outcome.Error); + Assert.Contains("captcha held it", outcome.Error); + } + + private static AgentPacket OnePacket(params (string Id, string Label, string Answer)[] fields) => new() + { + ApplicationName = "acme-engineer.md", Provider = "unknown", + Questions = [.. fields.Select(f => new PacketQuestion(f.Id, f.Label, true, PacketQuestion.Text, [], PacketQuestion.Standard))], + Answers = fields.ToDictionary(f => f.Id, f => f.Answer), + }; + + [SkippableFact] + public async Task A_submit_that_is_a_scripts_anchor_inside_the_form_is_clicked() + { + // Tential: APPLY NOW ends the form, and every run + // filled it and said "no Submit button found" (#424). The posting's own APPLY NOW, outside + // the form, is never the one clicked. + Skip.IfNot(Available, "Node Playwright is not installed (npm ci)"); + var outcome = await Submitter().RunAsync($"{_fixtureUrl}/jobs/anchor-submit", + OnePacket(("first_name", "First Name", "Ada")), null, dryRun: false); + Assert.True(outcome.Submitted, outcome.Error); + Assert.Equal("Ada", Assert.Single(_posts)["job_application[first_name]"]); + } + + [SkippableFact] + public async Task A_select_required_only_by_the_star_in_its_label_and_left_on_its_placeholder_is_handed_to_the_drafter() + { + // HubSpot: a dozen selects marked "… *" and nothing else, each on "-- Select" (a placeholder + // with a value). The sweep let them through, Submit stayed disabled, "no Submit button + // found" (#425). Now they are unmapped and go to the drafter, placeholder not an option. + Skip.IfNot(Available, "Node Playwright is not installed (npm ci)"); + var dry = await Submitter().RunAsync($"{_fixtureUrl}/jobs/starred-selects", + OnePacket(("first_name", "First Name", "Ada")), null, dryRun: true); + Assert.Contains("question_45", dry.Unmapped); + Assert.Contains("question_54", dry.Unmapped); + Assert.DoesNotContain("question_53", dry.Unmapped); // no star: not required + Assert.NotNull(dry.Discovered); + var auth = Assert.Single(dry.Discovered!, q => q.Id == "question_45"); + Assert.True(auth.Required); + Assert.Equal(["Yes", "No"], auth.Options); + + var real = await Submitter().RunAsync($"{_fixtureUrl}/jobs/starred-selects", + OnePacket(("first_name", "First Name", "Ada")), null, dryRun: false); + Assert.False(real.Submitted); + Assert.StartsWith("refused to submit: required fields could not be mapped", real.Error); + Assert.Empty(_posts); + + // Drafted, the next run fills them — a "-- Select" with a value is not taken for an + // answer the board already holds — and Submit, enabled by then, goes. + var drafted = OnePacket(("first_name", "First Name", "Ada")); + drafted.Questions.AddRange(dry.Discovered!); + drafted.Answers["question_45"] = "Yes"; + drafted.Answers["question_54"] = "Central"; + var done = await Submitter().RunAsync($"{_fixtureUrl}/jobs/starred-selects", drafted, null, dryRun: false); + Assert.True(done.Submitted, done.Error); + var post = Assert.Single(_posts); + Assert.Equal("Yes", post["question_45"]); + Assert.Equal("Central", post["question_54"]); + } + + [SkippableFact] + public async Task An_apply_inside_an_embedded_posting_frame_is_found_and_pressed() + { + // Voleon embeds its Ashby posting in an iframe; the Apply in it was never looked for, and + // the run said "no Apply button or link on the page" (#426). + Skip.IfNot(Available, "Node Playwright is not installed (npm ci)"); + var outcome = await Submitter().RunAsync($"{_fixtureUrl}/jobs/embedded-posting", Packet(), (Pdf, "resume.pdf"), dryRun: true); + Assert.True(outcome.Filled, outcome.Error); + Assert.Contains("first_name", outcome.Mapped); + Assert.Empty(_posts); + } + + [SkippableFact] + public async Task Ashby_graphql_submit_answer_is_the_verdict_when_the_page_says_nothing() + { + Skip.IfNot(Available, "Node Playwright is not installed (npm ci)"); + // Nuclear Promise X: "no confirmation" with two 200s from non-user-graphql and nothing to + // tell sent from refused (#421). + var packet = new AgentPacket + { + ApplicationName = "acme-engineer.md", Provider = "ashby", + Questions = [new("first_name", "First Name", true, PacketQuestion.Text, [], PacketQuestion.Standard)], + Answers = new() { ["first_name"] = "Ada" }, + }; + var accepted = await Submitter().RunAsync($"{_fixtureUrl}/jobs/graphql-accepted", packet, null, dryRun: false); + Assert.True(accepted.Submitted, accepted.Error); + Assert.Contains("FormSubmitSuccess", accepted.Confirmation); + + var refused = await Submitter().RunAsync($"{_fixtureUrl}/jobs/graphql-refused", packet, null, dryRun: false); + Assert.False(refused.Submitted); + Assert.Contains("(ApiSubmitSingleApplicationFormAction) → 200", refused.Error); + Assert.Contains("the board said \"Missing required field: Location\"", refused.Error); + } + + [SkippableFact] + public async Task An_apply_that_opens_another_posting_in_a_noopener_tab_is_followed_through_its_own_apply() + { + // 7seventy: Apply opens the employer's posting in a new tab (rel=noopener). The new tab's + // first request had no frame yet, asking for one threw in the route handler, and the tab + // hung; and once it loads, that posting's own Apply has to be pressed too (#426). + Skip.IfNot(Available, "Node Playwright is not installed (npm ci)"); + var outcome = await Submitter().RunAsync($"{_fixtureUrl}/jobs/aggregator", Packet(), (Pdf, "resume.pdf"), dryRun: true); + Assert.True(outcome.Filled, outcome.Error); + Assert.Contains("first_name", outcome.Mapped); + Assert.EndsWith("/jobs/1", outcome.Url); + Assert.Empty(_posts); + } + + [Theory] + [InlineData("Apply Now", true)] + [InlineData("Postuler maintenant", true)] + [InlineData("Postular", true)] + [InlineData("Candidater", true)] + [InlineData("Candidatar-se", true)] + [InlineData("Candidate FEEDBACK", false)] + [InlineData("Candidate Experience", false)] + [InlineData("Candidates", false)] + [InlineData("Apply later", false)] + public void The_apply_trigger_is_a_word_for_applying_not_the_english_word_candidate(string name, bool trigger) => + // Ford's "Candidate FEEDBACK" tab was pressed as Apply and opened a survey (#423). + Assert.Equal(trigger, BrowserSession.IsApplyTrigger(name)); + + [SkippableFact] + public async Task A_candidate_feedback_tab_is_never_pressed_for_apply() + { + Skip.IfNot(Available, "Node Playwright is not installed (npm ci)"); + var outcome = await Submitter().RunAsync($"{_fixtureUrl}/jobs/candidate-tab", Packet(), (Pdf, "resume.pdf"), dryRun: true); + Assert.True(outcome.Filled, outcome.Error); + Assert.Contains("first_name", outcome.Mapped); + } + + [Theory] + [InlineData("Principal AI Engineer This job has expired Sorry, this job has expired SHARE THIS JOB")] // SmartRecruiters + [InlineData("C# Developer | $65/hr Remote We are no longer accepting new profiles for this position.")] // Ceipal + [InlineData("The employer has taken the job offline. The operation failed. Try again")] // MeeBoss + public void Expired_and_offline_postings_read_as_closed(string text) => + // Each read as "no Apply button or link on the page" (#427). + Assert.True(BrowserSubmitter.IsClosedPosting(text)); + + [SkippableFact] + public async Task A_bot_protection_block_page_is_the_site_refusing_this_server_at_any_length() + { + // remotehunter.com: Cloudflare's 403 page is over 600 characters, and read as "no Apply + // button or link on the page" (#428). + Skip.IfNot(Available, "Node Playwright is not installed (npm ci)"); + var outcome = await Submitter().RunAsync($"{_fixtureUrl}/jobs/cloudflare-block", Packet(), (Pdf, "resume.pdf"), dryRun: true); + Assert.Contains("refused this server (HTTP 403)", outcome.Error); + Assert.False(outcome.Closed); + } + + [SkippableFact] + public async Task An_email_sign_in_met_mid_wizard_is_named_and_its_next_never_pressed() + { + // ClearCompany: after page one, a returning applicant gets "Enter the email you used to + // apply" (a plain text box) and NEXT. The walker took it for a form page, pressed NEXT on it + // empty, and the run ended "no Submit button found — NEXT, close" (#429). + Skip.IfNot(Available, "Node Playwright is not installed (npm ci)"); + var outcome = await Submitter().RunAsync($"{_fixtureUrl}/jobs/returning-applicant", + OnePacket(("first_name", "First Name", "Ada"), ("email", "Email", "ada@example.com")), null, dryRun: false); + Assert.False(outcome.Submitted); + Assert.StartsWith("this page is an email sign-in", outcome.Error); + Assert.Empty(_posts); + } + private static string FindRepoRoot() { var dir = new DirectoryInfo(AppContext.BaseDirectory); diff --git a/api/ApplyTrack.Api/Agent/AnswerDrafter.cs b/api/ApplyTrack.Api/Agent/AnswerDrafter.cs index 753b9ec..9166df0 100644 --- a/api/ApplyTrack.Api/Agent/AnswerDrafter.cs +++ b/api/ApplyTrack.Api/Agent/AnswerDrafter.cs @@ -37,13 +37,17 @@ public sealed partial class AnswerDrafter // deterministic branch at all — it fell through to the model, which answered it from // the brief's saved figure with no units check. Keep this list generous: a salary // question that is NOT recognised skips every guard below. - [GeneratedRegex(@"salary|compensation|remunerat|renumerat|pay (?:expectation|range|rate)|desired pay|rate expectation|expected (?:pay|rate)|day rate", RegexOptions.IgnoreCase)] + // "hourly rate" too: Wise Integration's "expected hourly rate for this role?" matched none + // of these, and fell to the website rule below (#422). + [GeneratedRegex(@"salary|compensation|remunerat|renumerat|pay (?:expectation|range|rate)|desired pay|rate expectation|expected (?:pay|rate)|day rate|hourly (?:rate|pay)|rate per hour", RegexOptions.IgnoreCase)] private static partial Regex Salary(); [GeneratedRegex(@"linkedin", RegexOptions.IgnoreCase)] private static partial Regex LinkedIn(); [GeneratedRegex(@"github", RegexOptions.IgnoreCase)] private static partial Regex GitHub(); - [GeneratedRegex(@"portfolio|website|personal site|url", RegexOptions.IgnoreCase)] + // "URL" as a word: bare, it matched inside "hoURLy" and an hourly-rate question was + // answered with the LinkedIn link (#422). + [GeneratedRegex(@"portfolio|website|personal site|\burls?\b", RegexOptions.IgnoreCase)] private static partial Regex Website(); [GeneratedRegex(@"facebook|twitter|instagram|mastodon|bluesky|threads", RegexOptions.IgnoreCase)] private static partial Regex Social(); @@ -404,6 +408,12 @@ public static string MoneyCurrency(string text) => public static string Country(AnswerContext ctx) => ctx.Settings.Country.Length > 0 ? ctx.Settings.Country : CountryFromLocation(ctx.Resume.Location); + /// The country an alias stands for ("USA", "United States of America" → "United + /// States"), or the text as given when it is no known alias — the form + /// reads its aliases from (#432). + public static string CanonicalCountry(string text) => + CountryNames.TryGetValue(text.Trim(), out var canonical) ? canonical : text.Trim(); + /// The option in a fixed country list that means : the /// same name, a known alias ("USA", "United States of America"), or a comma-joined option /// one of whose parts is either. Public for tests. diff --git a/api/ApplyTrack.Api/Agent/AtsProvider.cs b/api/ApplyTrack.Api/Agent/AtsProvider.cs index 9da1350..e235ecb 100644 --- a/api/ApplyTrack.Api/Agent/AtsProvider.cs +++ b/api/ApplyTrack.Api/Agent/AtsProvider.cs @@ -283,6 +283,50 @@ public static string ApplyUrl(string link, string provider) return $"https://{host}/embed/job_app?for={Uri.EscapeDataString(board)}&token={m.Groups[1].Value}"; } + // Ashby's embed parameter on an employer's own careers page: ?ashby_jid=. + [GeneratedRegex(@"[?&]ashby_jid=([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})", RegexOptions.IgnoreCase)] + private static partial Regex AshbyEmbed(); + + // The organisation as a page that embeds Ashby names it: jobs.ashbyhq.com/, or the + // embed script's api.ashbyhq.com/posting-api/job-board/. + [GeneratedRegex(@"(?:jobs\.ashbyhq\.com|ashbyhq\.com/posting-api/job-board)/([A-Za-z0-9][A-Za-z0-9._-]*)", RegexOptions.IgnoreCase)] + private static partial Regex AshbyOrgRef(); + + /// Does the link name an Ashby job on a page that is not Ashby's own? + public static bool HasAshbyJobId(string link) => + AshbyEmbed().IsMatch(link ?? "") + && Uri.TryCreate(link, UriKind.Absolute, out var u) && !u.Host.EndsWith("ashbyhq.com", StringComparison.OrdinalIgnoreCase); + + /// + /// Ashby's hosted application for an employer page that names an Ashby job with + /// ?ashby_jid= — https://jobs.ashbyhq.com/{org}/{jid}/application — or null when + /// the link names none. The organisation is read off the page's HTML when it names one, else + /// taken from the site's own name (hercules.app → hercules), which is how Ashby + /// boards are usually named; the caller keeps the hosted page only if a form renders there (#434). + /// + public static string? AshbyHostedForm(string link, string html = "") + { + if (!HasAshbyJobId(link) || !Uri.TryCreate(link, UriKind.Absolute, out var uri)) return null; + var jid = AshbyEmbed().Match(link).Groups[1].Value.ToLowerInvariant(); + var org = ""; + foreach (Match m in AshbyOrgRef().Matches(html ?? "")) + { + var candidate = m.Groups[1].Value; + // A link to the board's own api or embed script, not an organisation. + if (candidate.Equals("api", StringComparison.OrdinalIgnoreCase) || candidate.Equals("embed", StringComparison.OrdinalIgnoreCase)) continue; + org = candidate; + break; + } + if (org.Length == 0) + { + var labels = uri.Host.ToLowerInvariant().Split('.'); + var i = 0; + while (i < labels.Length - 2 && labels[i] is "www" or "careers" or "career" or "jobs" or "job" or "apply" or "work") i++; + org = labels.Length >= 2 ? labels[i] : ""; + } + return org.Length == 0 ? null : $"https://jobs.ashbyhq.com/{Uri.EscapeDataString(org)}/{jid}/application"; + } + /// Board token + job id for a Greenhouse posting, from the hosted URL or an /// embed's gh_jid plus the poller's auto:greenhouse:{board} source. public static bool TryParseGreenhouse(string link, string source, out string board, out string jobId) diff --git a/api/ApplyTrack.Api/Agent/Browser/BrowserSession.cs b/api/ApplyTrack.Api/Agent/Browser/BrowserSession.cs index 626ba26..9d41303 100644 --- a/api/ApplyTrack.Api/Agent/Browser/BrowserSession.cs +++ b/api/ApplyTrack.Api/Agent/Browser/BrowserSession.cs @@ -27,10 +27,20 @@ public sealed partial class BrowserSession : IAsyncDisposable ]; // Localised too: join.com renders its posting in the employer's language, so the - // real Apply on a French posting is "Postuler maintenant" (#210). - [GeneratedRegex(@"^\s*(?:apply\b|i['’]?m interested|postuler\b|(?:jetzt )?bewerben|aplicar\b|solicitar\b|candidat)", RegexOptions.IgnoreCase)] + // real Apply on a French posting is "Postuler maintenant" (#210). Spanish boards say + // "Postular" (laborum.news). "candidat" is the verb or noun of the Romance languages — + // candidater, candidatar, candidatura — never the English word: Ford's careers page has + // a "Candidate FEEDBACK" tab, the reveal pressed it as Apply, and a survey opened where + // the form should have (#423). + // "Easy Apply" and its kin as well: N-iX folds its whole form behind an "Easy Apply" + // button, which "begins with apply" never matched, so the form stayed hidden and only its + // file input was filled (#418). + [GeneratedRegex(@"^\s*(?:(?:easy|quick|one[- ]click)\s+apply\b|apply\b|i['’]?m interested|postuler\b|postular\b|(?:jetzt )?bewerben|aplicar\b|solicitar\b|candidat(?!es?\b))", RegexOptions.IgnoreCase)] private static partial Regex ApplyTrigger(); + /// Does a control's name read as the way into an application? Public for tests. + public static bool IsApplyTrigger(string name) => ApplyTrigger().IsMatch(name) && !LaterWords().IsMatch(name); + /// /// A control that only LOOKS like the way in: "Apply later", "Send me the link", /// "Remind me", a job-alert subscription. join.com puts an email box and an @@ -61,7 +71,50 @@ public sealed partial class BrowserSession : IAsyncDisposable if (/(?:^|[^a-z])(?:search(?!select)|keyword)|(?:location|job|keyword)search|createnewalert|(?:^|[^a-z])frequency(?:$|[^a-z])|job.?alert|subscribe/.test(own)) return true; return !!el.closest('[role=search], form[action*="search" i], form[class*="search" i], form[id*="search" i], form[name*="search" i], ' + 'form[action*="subscribe" i], form[class*="subscribe" i], form[id*="subscribe" i], form[name*="subscribe" i], ' - + 'form[action*="alert" i], form[class*="alert" i], form[id*="alert" i], [class*="job-alert" i], [class*="jobalert" i], [id*="jobalert" i], [class*="emailsubscribe" i]'); + + 'form[action*="alert" i], form[class*="alert" i], form[id*="alert" i], [class*="job-alert" i], [class*="jobalert" i], [id*="jobalert" i], [class*="emailsubscribe" i], ' + // A newsletter signup in the footer: Tecbrains' Mailchimp for WordPress box + // (
) was discovered as the + // application's "email" and left unmapped on every dry run (#419). Mailchimp's own + // embed too, and any form that calls itself a newsletter. + + 'form.mc4wp-form, #mc_embed_signup, form[action*="list-manage.com" i], ' + + 'form[class*="newsletter" i], form[id*="newsletter" i], form[action*="newsletter" i], [class*="newsletter-form" i], [class*="newsletter_form" i]'); + } + """; + + /// + /// A JavaScript predicate, (text) => bool: is this text a file dropzone's instructions + /// — "Click or drag & drop PDF", "Drop files here", "Browse files" — and nothing that names + /// the document? Ethos wraps its résumé input in such a label, beside the field's real title + /// "CV / Resume", and the question was discovered as "Click or drag & drop PDF" (#417). + /// + public const string DropzoneJs = """ + (t) => /\b(?:drag\s*(?:&|and|'?n'?|or)?\s*drop|drop\s+(?:your\s+|a\s+|the\s+)?files?\s+here|click\s+(?:here\s+)?to\s+(?:upload|browse|attach|select)|browse\s+(?:for\s+)?(?:a\s+)?files?)/i.test(t) + && !/resume|résumé|\bcv\b|cover|letter|portfolio|transcript|photo|certificat/i.test(t) + """; + + /// The same test as , for a packet question's label (#417). + public static bool IsDropzoneText(string text) => + Regex.IsMatch(text, @"\b(?:drag\s*(?:&|and|'?n'?|or)?\s*drop|drop\s+(?:your\s+|a\s+|the\s+)?files?\s+here|click\s+(?:here\s+)?to\s+(?:upload|browse|attach|select)|browse\s+(?:for\s+)?(?:a\s+)?files?)", RegexOptions.IgnoreCase) + && !Regex.IsMatch(text, @"resume|résumé|\bcv\b|cover|letter|portfolio|transcript|photo|certificat", RegexOptions.IgnoreCase); + + /// + /// A JavaScript function, (selector) => Element[]: querySelectorAll over the + /// document and every open shadow root in it. Manatal's careers pages (careers-page.com) + /// draw the whole application inside a web component's shadow root, so every page script that + /// read document.querySelectorAll('input, …') saw a page with no form: discovery found no + /// questions, the packet fell back to the standard set, and the résumé that did attach was + /// never seen to have taken — "required fields could not be mapped (std:resume)" (yo-ai-labs, #436). + /// Playwright's own locators already pierce shadow roots; the page scripts now agree with them. + /// + public const string DeepJs = """ + (sel) => { + const out = []; + const walk = root => { + out.push(...root.querySelectorAll(sel)); + for (const host of root.querySelectorAll('*')) if (host.shadowRoot) walk(host.shadowRoot); + }; + walk(document); + return out; } """; @@ -209,6 +262,15 @@ await context.RouteAsync("**/*", async route => await route.AbortAsync(); return; } + // Display advertising is never part of an application, and Google's AdSense "vignette" + // is worse than noise: a full-screen ad that 7seventy's job aggregator raises over the + // page when its Apply link is clicked (#google_vignette), so the click opened an ad, + // not the posting, and the run reported "no form appeared" (#426). + if (IsAdNetwork(u.Host)) + { + await route.AbortAsync(); + return; + } // The live-form verification harness: nothing that could create an application // leaves. The one exception is the board's résumé uploader, which stages the file // straight to object storage with a multipart POST — an application is always @@ -222,7 +284,7 @@ await context.RouteAsync("**/*", async route => // that hosts its form are fine); sub-resources are left alone so the form // can load its scripts and styles. Any top-level frame in the context — the // tab an Apply link opens is held to the same rule as the first page. - if (req.IsNavigationRequest && req.Frame.ParentFrame is null && !HostAllowed(u.Host, allowedHost) + if (req.IsNavigationRequest && IsTopLevel(req) && !HostAllowed(u.Host, allowedHost) && BoardAccount.For(session._accounts, u.Host) is null) { lock (session._refused) { session._refused.Add(u.Host); session._refusedUrls.Add(u.AbsoluteUri); } @@ -336,6 +398,83 @@ public static async Task DismissConsentAsync(IPage page) return false; } + /// + /// For a link that names an Ashby job on an employer's own page (?ashby_jid=) where no + /// application form shows: open Ashby's hosted copy of the form and hand that session back + /// instead, disposing — but only if a form renders there; otherwise + /// is returned as it was. Hercules' careers page takes the parameter + /// and lists its jobs without opening the one named, and the run reported "nothing on this + /// form could be filled" against the site's filters (#434). + /// + public static async Task AshbyHostedInsteadAsync(BrowserOptions options, BrowserSession opened, string link, + CancellationToken ct, IReadOnlyList? accounts = null) + { + if (!AtsProvider.HasAshbyJobId(link) || await WaitForApplicationFormAsync(opened.Page, 5_000)) + return opened; + string html; + try { html = await opened.Page.ContentAsync(); } + catch (PlaywrightException) { html = ""; } + if (AtsProvider.AshbyHostedForm(link, html) is not { } hosted) + return opened; + BrowserSession? alt = null; + try + { + alt = await OpenAsync(options, hosted, ct, accounts); + if (await WaitForApplicationFormAsync(alt.Page, 10_000)) + { + await opened.DisposeAsync(); + return alt; + } + } + catch (PlaywrightException) { /* the hosted copy would not open; keep the employer's page */ } + catch (TimeoutException) { /* ditto */ } + if (alt is not null) await alt.DisposeAsync(); + return opened; + } + + // The one button of a notice that only wants to be read: OK, Got it, Understood, Close. + [GeneratedRegex(@"^\s*(?:ok(?:ay)?|got it|understood|dismiss|close)\s*[.!]?\s*$", RegexOptions.IgnoreCase)] + private static partial Regex NoticeButton(); + + /// + /// Press the OK of a modal notice that sits over the form and asks for nothing: a dialog + /// with no field in it and one button, and that button an OK. Flexhire opens "Apply to Job — + /// this job accepts applicants located in … We couldn't detect your location" over its form + /// on Apply; the run filled the boxes under it by their ids, but the "Upload Resume/CV" click + /// landed on the dialog's backdrop, no résumé went up, and the board answered "Resume is + /// required" (#330, #437). A dialog that carries a field or any second button is a question, + /// and is left alone. True when one was pressed. + /// + public static async Task AcknowledgeNoticeAsync(IPage page) + { + foreach (var frame in page.Frames) + { + try + { + var dialogs = frame.Locator("[role=dialog]:visible, [role=alertdialog]:visible, [aria-modal=true]:visible"); + var n = Math.Min(await dialogs.CountAsync(), 4); + for (var i = 0; i < n; i++) + { + var d = dialogs.Nth(i); + if (await d.Locator("input:not([type=hidden]):not([type=button]):not([type=submit]), select, textarea").CountAsync() > 0) continue; + // Its buttons: the OK, and nothing else but links set in its sentences (Flexhire's + // "and 21 more countries" is a button inside the notice's paragraph). + var buttons = d.Locator("button:visible, [role=button]:visible, input[type=button]:visible, input[type=submit]:visible") + .Filter(new() { HasNot = frame.Locator("xpath=ancestor::p | ancestor::li") }); + if (await buttons.CountAsync() != 1) continue; + var name = (await buttons.First.InnerTextAsync(new() { Timeout = 1_000 })).Trim(); + if (!NoticeButton().IsMatch(name)) continue; + await buttons.First.ClickAsync(new() { Timeout = 2_000 }); + await page.WaitForTimeoutAsync(400); + return true; + } + } + catch (TimeoutException) { /* the notice went away on its own */ } + catch (PlaywrightException) { /* a frame mid-navigation */ } + } + return false; + } + /// The cookies the context holds for — how a signed-in /// portal session is read back off the browser (#221). public Task> CookiesAsync(string url) => _context.CookiesAsync([url]); @@ -421,6 +560,28 @@ private static bool IsStorageUpload(Uri u, IRequest req) => && req.Headers.TryGetValue("content-type", out var ct) && ct.StartsWith("multipart/form-data", StringComparison.OrdinalIgnoreCase); + /// + /// Is a navigation request a top-level page's? The first request of a new tab — the one a + /// target="_blank" Apply opens — is issued before its frame exists, and asking for that frame + /// throws ("Frame for this navigation request is not available"). It threw inside the route + /// handler, the request was never continued, and the tab hung on about:blank: 7seventy's + /// "Apply for This Job" (rel=noopener, to Ashby) opened nothing the run could see (#426). A + /// request with no frame yet is a new tab's, which is top-level. + /// + private static bool IsTopLevel(IRequest req) + { + try { return req.Frame.ParentFrame is null; } + catch (PlaywrightException) { return true; } + } + + /// An ad network's host — Google's AdSense and DoubleClick (#426). + private static bool IsAdNetwork(string host) + { + host = host.ToLowerInvariant(); + return host.EndsWith("googlesyndication.com") || host.EndsWith("doubleclick.net") + || host.EndsWith("googleadservices.com") || host.StartsWith("adservice.google."); + } + public static bool HostAllowed(string host, string original) { host = host.ToLowerInvariant(); @@ -437,10 +598,11 @@ public static bool HostAllowed(string host, string original) const widget = __WIDGET__; const shown = el => { const r = el.getBoundingClientRect(); const s = getComputedStyle(el); return r.width > 0 && r.height > 0 && s.visibility !== 'hidden' && s.display !== 'none'; }; - return [...document.querySelectorAll('input[type=file], input[type=text], input[type=email], input:not([type]), textarea, select')] + const deep = __DEEP__; + return deep('input[type=file], input[type=text], input[type=email], input:not([type]), textarea, select') .some(el => !widget(el) && (el.getAttribute('type') === 'file' || shown(el))); } - """.Replace("__WIDGET__", WidgetJs); + """.Replace("__WIDGET__", WidgetJs).Replace("__DEEP__", DeepJs); /// Is a form control on screen — in the page, or in any frame it embeds? A /// search or job-alert widget's controls never count (#214). @@ -511,8 +673,9 @@ private static async Task AnyFrameAsync(IPage page, string script) const widget = __WIDGET__; const shown = el => { const r = el.getBoundingClientRect(); const s = getComputedStyle(el); return r.width > 0 && r.height > 0 && s.visibility !== 'hidden' && s.display !== 'none'; }; + const deep = __DEEP__; let texts = 0; - for (const el of document.querySelectorAll('input, textarea, select')) { + for (const el of deep('input, textarea, select')) { if (widget(el)) continue; // A code entered a character per box is a verification step, not the form: // Oracle's six pin-code boxes read as "the form" the moment they appeared (#318). @@ -530,11 +693,11 @@ private static async Task AnyFrameAsync(IPage page, string script) private static readonly string ApplicationFormScript = ApplicationFormScriptTemplate .Replace("__FILE_RULE__", "if (type === 'file') return true;") - .Replace("__WIDGET__", WidgetJs); + .Replace("__WIDGET__", WidgetJs).Replace("__DEEP__", DeepJs); private static readonly string RevealedFormScript = ApplicationFormScriptTemplate .Replace("__FILE_RULE__", "if (type === 'file' && shown(el)) return true;") - .Replace("__WIDGET__", WidgetJs); + .Replace("__WIDGET__", WidgetJs).Replace("__DEEP__", DeepJs); /// /// Oracle Recruiting's "easy apply" is not the form: signed in, it opens on "Let's make this @@ -597,30 +760,7 @@ public async Task RevealFormAsync() await KeepPostingTextAsync(); return; } - // By accessible name first, then by what the control visibly says. Allstate labels its - // link aria-label="Lead .Net Software Engineer Apply Now open in new window": the name - // opens with the job title, so nothing "began with apply" and the run reported "no Apply - // button or link on the page" with "Apply now" in plain view (#280). And the first one - // that can be SEEN, not the first in the document — a collapsed panel's own "Apply" - // must not stand in for the posting's. - ILocator? apply = null; - foreach (var matches in new[] - { - Page.GetByRole(AriaRole.Button, new() { NameRegex = ApplyTrigger() }).Filter(new() { HasNotTextRegex = LaterWords() }), - Page.GetByRole(AriaRole.Link, new() { NameRegex = ApplyTrigger() }).Filter(new() { HasNotTextRegex = LaterWords() }), - Page.Locator("a[href], button, [role=button], input[type=submit], input[type=button]") - .Filter(new() { HasTextRegex = ApplyTrigger() }).Filter(new() { HasNotTextRegex = LaterWords() }), - }) - { - try - { - var count = Math.Min(await matches.CountAsync(), 8); - for (var i = 0; i < count && apply is null; i++) - if (await matches.Nth(i).IsVisibleAsync()) apply = matches.Nth(i); - } - catch (PlaywrightException) { /* next */ } - if (apply is not null) break; - } + var apply = await FindApplyAsync(); if (apply is null) { // A page whose only controls are radio/checkbox questions and a Continue is a @@ -633,6 +773,77 @@ public async Task RevealFormAsync() return; } + // The posting, while it is still on screen: Apply replaces it. + await KeepPostingTextAsync(); + // Apply may lead to another posting page with its own Apply: an aggregator (7seventy) + // opens the employer's Ashby posting in a new tab, and that page's "Apply for this Job" is + // the way to the form. Nobody pressed it, and the run said "no form appeared" (#426). + // One more hop, only onto a page Apply actually led to — never the same page twice. + string applyHost = ""; + for (var hop = 0; ; hop++) + { + var outcome = await PressApplyAsync(apply, canHop: hop == 0); + if (outcome.Done) return; + if (outcome.ApplyHost.Length > 0 && applyHost.Length == 0) applyHost = outcome.ApplyHost; + if (hop == 0 && outcome.Moved && await FindApplyAsync() is { } again) + { + apply = again; + continue; + } + break; + } + var landedAt = Uri.TryCreate(Page.Url, UriKind.Absolute, out var at) ? at.Host : ""; + RevealNote = NoFormNote(Refused, landedAt, applyHost); + if (AccountOnlyAts(landedAt) is not null) NeedsAccountAt = landedAt; + else if (Refused.Count > 0 && AccountOnlyAts(Refused[0]) is not null) NeedsAccountAt = Refused[0]; + else if (AccountOnlyAts(applyHost) is not null) NeedsAccountAt = applyHost; + } + + /// + /// The visible way into the application, or null. By accessible name first, then by what the + /// control visibly says. Allstate labels its link aria-label="Lead .Net Software Engineer Apply + /// Now open in new window": the name opens with the job title, so nothing "began with apply" + /// and the run reported "no Apply button or link on the page" with "Apply now" in plain view + /// (#280). And the first one that can be SEEN, not the first in the document — a collapsed + /// panel's own "Apply" must not stand in for the posting's. The top document first, then the + /// frames it embeds: Voleon's careers page embeds its Ashby posting in an iframe, and the + /// posting's "Apply for this Job" is in there, out of reach of a search of the page (#426). + /// + private async Task FindApplyAsync() + { + foreach (var frame in new[] { Page.MainFrame }.Concat(Page.Frames.Where(f => f != Page.MainFrame))) + { + foreach (var matches in new[] + { + frame.GetByRole(AriaRole.Button, new() { NameRegex = ApplyTrigger() }).Filter(new() { HasNotTextRegex = LaterWords() }), + frame.GetByRole(AriaRole.Link, new() { NameRegex = ApplyTrigger() }).Filter(new() { HasNotTextRegex = LaterWords() }), + frame.Locator("a[href], button, [role=button], input[type=submit], input[type=button]") + .Filter(new() { HasTextRegex = ApplyTrigger() }).Filter(new() { HasNotTextRegex = LaterWords() }), + }) + { + try + { + var count = Math.Min(await matches.CountAsync(), 8); + for (var i = 0; i < count; i++) + if (await matches.Nth(i).IsVisibleAsync()) return matches.Nth(i); + } + catch (PlaywrightException) { /* next */ } + } + } + return null; + } + + /// What one press of Apply came to: Done when the reveal is settled (a form, + /// a sign-in, a wall, or a click that failed — says which); + /// otherwise nothing appeared, and Moved says whether Apply led to another page or tab. + private readonly record struct ApplyPress(bool Done, bool Moved, string ApplyHost); + + /// An address without its #fragment: "moved" means another tab or another address, + /// not a fragment an overlay or an in-page anchor set on the same page (#426). + private static string Unfragmented(string url) => url.Split('#')[0]; + + private async Task PressApplyAsync(ILocator apply, bool canHop) + { // Where Apply points, read before the click: Allstate's is a target="_blank" anchor to // Workday, and when the tab it opens never reaches this context the run had nothing to // go on and said "no form appeared" about a page it had never left (#280). The href @@ -647,8 +858,10 @@ public async Task RevealFormAsync() } catch (PlaywrightException) { /* a button, not a link */ } - // The posting, while it is still on screen: Apply replaces it. - await KeepPostingTextAsync(); + var urlBefore = Page.Url; + // The tabs already open are not what this Apply opened (the posting Apply led away from, + // on a second hop). + var known = _context.Pages.ToHashSet(); var popup = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); void OnPage(object? _, IPage p) => popup.TrySetResult(p); _context.Page += OnPage; @@ -670,13 +883,13 @@ public async Task RevealFormAsync() if (attempt == 0 && await DismissConsentAsync(Page)) continue; _context.Page -= OnPage; RevealNote = "the Apply button did not respond within 5 s"; - return; + return new(true, false, applyHost); } catch (PlaywrightException ex) { _context.Page -= OnPage; RevealNote = "clicking Apply failed: " + ex.Message.Split('\n')[0]; - return; + return new(true, false, applyHost); } } // Gainwell's "Apply Now" only opens a menu — "Apply Now" / "Start applying with LinkedIn" — @@ -696,6 +909,7 @@ public async Task RevealFormAsync() // Apply opens its sign-in in a new tab only after a round trip, later than the moment // after the click, and a run that kept watching the posting saw nothing (#235). var deadline = DateTime.UtcNow.AddSeconds(10); + var settled = DateTime.UtcNow.AddSeconds(5); var adopted = false; try { @@ -705,7 +919,7 @@ public async Task RevealFormAsync() // handler was attached — or one the event never fired for — is still in the // context, so the context's own list is asked too. if (!adopted && !popup.Task.IsCompletedSuccessfully - && _context.Pages.FirstOrDefault(p => p != Page && !p.IsClosed) is { } stray) + && _context.Pages.FirstOrDefault(p => !known.Contains(p) && !p.IsClosed) is { } stray) popup.TrySetResult(stray); if (!adopted && popup.Task.IsCompletedSuccessfully) { @@ -718,13 +932,14 @@ public async Task RevealFormAsync() // A new tab is a whole app booting through the proxy: Alignerr's goes /signup → // /signin behind a spinner and showed its wall well after ten seconds on pluto. deadline = DateTime.UtcNow.AddSeconds(30); + settled = DateTime.UtcNow.AddSeconds(5); } // A field on screen settles it — and only then is the sign-in question asked, so a // page still mid-navigation cannot answer "no sign-in" a moment before it renders one. if (await AnyFieldVisibleAsync(Page)) { if (await SignInFormVisibleAsync(Page)) await TrySignInAsync(); - return; + return new(true, true, applyHost); } // Alignerr and its kind: Apply opens an account sign-up that only offers "Continue // with Google / LinkedIn" — no username box, no form, nothing the browser can fill or @@ -734,18 +949,20 @@ public async Task RevealFormAsync() if (providers.Count > 0) { RevealNote = SocialWallNote(Page.Url, providers); - return; + return new(true, true, applyHost); } if (DateTime.UtcNow >= deadline) break; + // Landed on another posting page that has its own Apply and, after a few seconds to + // render, still no form: that Apply is the way on, and waiting out the thirty + // seconds a new tab is given only delays pressing it (#426). + if (canHop && DateTime.UtcNow >= settled && (adopted || Unfragmented(Page.Url) != Unfragmented(urlBefore)) + && await FindApplyAsync() is not null) + break; await Page.WaitForTimeoutAsync(500); } } finally { _context.Page -= OnPage; } - var landedAt = Uri.TryCreate(Page.Url, UriKind.Absolute, out var at) ? at.Host : ""; - RevealNote = NoFormNote(Refused, landedAt, applyHost); - if (AccountOnlyAts(landedAt) is not null) NeedsAccountAt = landedAt; - else if (Refused.Count > 0 && AccountOnlyAts(Refused[0]) is not null) NeedsAccountAt = Refused[0]; - else if (AccountOnlyAts(applyHost) is not null) NeedsAccountAt = applyHost; + return new(false, adopted || Unfragmented(Page.Url) != Unfragmented(urlBefore), applyHost); } /// diff --git a/api/ApplyTrack.Api/Agent/Browser/BrowserSubmitter.cs b/api/ApplyTrack.Api/Agent/Browser/BrowserSubmitter.cs index c2a5f0b..c2d743d 100644 --- a/api/ApplyTrack.Api/Agent/Browser/BrowserSubmitter.cs +++ b/api/ApplyTrack.Api/Agent/Browser/BrowserSubmitter.cs @@ -79,9 +79,16 @@ public sealed partial class BrowserSubmitter : IBrowserSubmitter // prose ("no longer supported", etc.) never trips it. // "Page not found" is what Greenhouse serves for a posting that has been taken down // outright (Cresteo and Varicent on 2026-09-13): no gone-notice, no form, a 404 page. - [GeneratedRegex(@"job you are looking for is no longer open|no longer (?:open|accepting applications)|(?:position|posting|job|role|opening) (?:has been|was|is now) (?:filled|closed)|this (?:position|posting|job|role|opening|opportunity) is (?:no longer available|closed|not available anymore|currently not available)|\bpage not found\b|\bjob (?:posting )?not found\b|this job (?:posting )?(?:is )?no longer exists", RegexOptions.IgnoreCase)] + // And three that read as "no Apply button" until they were taught (#427): SmartRecruiters' + // "Sorry, this job has expired", Ceipal's "We are no longer accepting new profiles for this + // position", MeeBoss's "The employer has taken the job offline". + [GeneratedRegex(@"job you are looking for is no longer open|no longer (?:open|accepting (?:new )?(?:applications|applicants|candidates|profiles|resumes))|this (?:job|posting|position|role) has expired|(?:taken|took) (?:the|this) (?:job|posting|position|role) offline|(?:position|posting|job|role|opening) (?:has been|was|is now) (?:filled|closed)|this (?:position|posting|job|role|opening|opportunity) is (?:no longer available|closed|not available anymore|currently not available)|\bpage not found\b|\bjob (?:posting )?not found\b|this job (?:posting )?(?:is )?no longer exists|\bcouldn['’]t find (?:that|this|the) (?:page|job|posting)\b|\b(?:job|posting|position|page) you(?:['’]re| are) looking for (?:could not|cannot|can['’]t) be found", RegexOptions.IgnoreCase)] private static partial Regex ClosedPosting(); + /// A bot-protection service's refusal page: Cloudflare, Akamai, Imperva, DataDome (#428). + [GeneratedRegex(@"you have been blocked|attention required!? \| cloudflare|access denied|request (?:was |has been )?blocked|pardon our interruption|verify you are (?:a )?human", RegexOptions.IgnoreCase)] + private static partial Regex BotBlock(); + [GeneratedRegex(@"/(?:home|careers?|jobs?|openings|positions|opportunities|search|job-?board)/?$", RegexOptions.IgnoreCase)] private static partial Regex JobListPath(); @@ -174,6 +181,9 @@ public async Task RunAsync( opened = await BrowserSession.OpenAsync(_options, form, ct, accounts); } } + // An employer page that names an Ashby job (?ashby_jid=) and shows no form for it: Ashby's + // own hosted copy is the form (Hercules, #434). + opened = await BrowserSession.AshbyHostedInsteadAsync(_options, opened, link, ct, accounts); await using var session = opened; var outcome = await RunOnPageAsync(session, link, packet, resumePdf, dryRun, ct, resumeText, coverLetter, awaitSecurityCode, accounts); // The run stopped at a sign-in no saved account covers: say which host, so the worker @@ -205,7 +215,10 @@ private async Task RunOnPageAsync( // The site will not talk to this server at all — McGraw Hill's careers site answers // pluto with a bare "403 Forbidden". That is not a page with no Apply button on it, // which is what the run used to report, and no retry will change it (#280). - if (session.Status is 401 or 403 or 429 or 451 && body.Length < 600) + // Cloudflare's own block page is longer than that, and remotehunter.com's read as "no + // Apply button or link on the page" (#428): a bot-protection block is the same verdict + // at any length. + if (session.Status is 401 or 403 or 429 or 451 && (body.Length < 600 || BotBlock().IsMatch(body))) { screenshot = await session.ScreenshotAsync(); return new SubmitOutcome(false, false, page.Url, "", screenshot, unmapped, mapped, @@ -239,6 +252,8 @@ private async Task RunOnPageAsync( // A consent banner that arrived with the form, or after the page went idle, would // sit over every field and make the first fill time out (#202). await BrowserSession.DismissConsentAsync(page); + // So would a notice that only wants its OK pressed (Flexhire's location notice, #437). + await BrowserSession.AcknowledgeNoticeAsync(page); // UKG (UltiPro): the sign-in the session just did lands a first-time candidate on // the employer's own profile step — name, phone, a privacy consent and "Create // account" — before the application. Filled by the generic pass, but the button is @@ -305,7 +320,11 @@ private async Task RunOnPageAsync( behindSignIn = true; } - foreach (var q in packet.Questions) + // Files first. A board that reads the uploaded résumé writes what it found into the + // fields — Robert Half's (Phenom) rewrote the phone already typed as "(571) · 297 · 5406" + // and refused it as "Invalid phone number format" (#438). Attached before the typing, + // the parse lands first and the packet's answers are what stays. + foreach (var q in packet.Questions.OrderBy(q => q.Type == PacketQuestion.File ? 0 : 1)) { ct.ThrowIfCancellationRequested(); // Already answered on a pre-screening step the run advanced past (#239): its @@ -322,21 +341,32 @@ private async Task RunOnPageAsync( } if (q.Type == PacketQuestion.File) { - if (IsResume(q) && await ResumeOnFileAsync(form)) + // A packet discovered before #417 named Ethos' résumé by its dropzone's + // instructions ("Click or drag & drop PDF"). On a form with one file input, + // a file field named by nothing but those instructions is that input: the résumé. + var isResume = IsResume(q) || (!IsCoverLetter(q) && BrowserSession.IsDropzoneText(q.Label) + && await OnlyFileInputAsync(form) is not null); + if (isResume && await ResumeOnFileAsync(form)) { // The candidate's account already holds a résumé and the form shows it // (SuccessFactors: "Upload Resume · resume.pdf · Edit document", #216). // Nothing to attach; the field is satisfied as it stands. mapped.Add(q.Id); } - else if (IsResume(q) && resumePdf is { } pdf) + else if (isResume && resumePdf is { } pdf) { // With a PDF on hand the résumé is required for the click whatever the // API said: Greenhouse's Job Board API lists GitLab's résumé as optional // while the rendered form marks it required, so an attach that failed // on an "optional" résumé landed in neither list and the run clicked // Submit into "Resume/CV is required" (#195). A failed attach refuses. - if (await AttachResumeAsync(form, q, pdf, resumeText)) mapped.Add(q.Id); + if (await AttachResumeAsync(form, q, pdf, resumeText)) + { + mapped.Add(q.Id); + // Let a résumé parse finish writing before anything is typed (#438). + try { await page.WaitForLoadStateAsync(LoadState.NetworkIdle, new() { Timeout = 5_000 }); } + catch (TimeoutException) { /* judged by what the fill finds */ } + } else unmapped.Add(q.Id); } else if (IsCoverLetter(q) && coverLetter.Length > 0) @@ -363,6 +393,10 @@ private async Task RunOnPageAsync( continue; } if (await FillAsync(form, q, answer)) mapped.Add(q.Id); + // A notice that came up under the fill takes the clicks meant for the field — + // Flexhire's location notice opens once Location is typed (#437). Acknowledged, + // the field gets one more go. + else if (await BrowserSession.AcknowledgeNoticeAsync(page) && await FillAsync(form, q, answer)) mapped.Add(q.Id); else if (q.Required) unmapped.Add(q.Id); } @@ -371,13 +405,20 @@ private async Task RunOnPageAsync( // last: when both halves went unmapped and such a box exists, it gets "First Last". await FillFullNameAsync(form, packet, mapped, unmapped); // Greenhouse's Education block, which its API never mentions (#332): from the résumé. + // What it fills is answered, whatever the packet's own answer for it did: Stripe's + // school--0 failed on the packet's "University of Maryland University College", the + // résumé's school then filled it, and the run still listed it unmapped (#433). foreach (var id in await FillGreenhouseEducationAsync(form, packet.Resume)) + { if (!mapped.Contains(id)) mapped.Add(id); + unmapped.Remove(id); + } // A résumé taken only through an "Upload Resume/CV" button — no file input until it is // pressed, so discovery never saw a question (Flexhire: "Resume is required", #330). if (resumePdf is { } buttonPdf && !packet.Questions.Any(q => IsResume(q) && mapped.Contains(q.Id)) && await UploadResumeByButtonAsync(form, buttonPdf)) mapped.Add("resume"); + await BrowserSession.AcknowledgeNoticeAsync(page); await RefillEmptiedAsync(form, packet, mapped, unmapped); // A form in pages — ClearCompany's "Page 1 · Page 2 · Page 3", evlo's five-step wizard // from "Upload Resume" to "Voluntary Self-Identification" — shows one page at a time @@ -476,6 +517,12 @@ private async Task RunOnPageAsync( continue; discovered.Add(live with { Required = true }); } + // A posting whose page drew its gone-notice after the first look — Torentify's + // "Job Not Found" arrives once its script has asked for the job — is closed, not a + // form nothing matched (#435). + if (mapped.Count == 0 && IsClosedPosting(await BodyTextAsync(page.MainFrame))) + return new SubmitOutcome(false, false, page.Url, "", screenshot, unmapped, mapped, + "posting is no longer open", Closed: true); if (mapped.Count == 0) return new SubmitOutcome(false, false, page.Url, "", screenshot, unmapped, mapped, AlreadyApplied().IsMatch(await BodyTextAsync(page.MainFrame)) @@ -507,12 +554,30 @@ private async Task RunOnPageAsync( var posts = new List(); var captchaRefused = false; var boardSaid = ""; + var boardAccepted = false; page.Response += (_, r) => { var req = r.Request; if (req.Method is "GET" or "HEAD" || IsChatter(r.Url)) return; if (Uri.TryCreate(r.Url, UriKind.Absolute, out var u)) - lock (posts) posts.Add($"{req.Method} {u.Host}{u.AbsolutePath} → {r.Status}"); + lock (posts) posts.Add($"{req.Method} {u.Host}{u.AbsolutePath}{GraphQlOp(u)} → {r.Status}"); + // Ashby answers every GraphQL call 200 and says in the body whether the + // application was taken: applicationFormResult is FormSubmitSuccess, or the form + // again (FormRender) with errors. Two runs ended "no confirmation" with nothing + // to say which (#421). Read the body: success is the confirmation, an error + // message is the board's own words. + if (r.Status is >= 200 and < 300 && u is not null && u.AbsolutePath.Contains("graphql", StringComparison.OrdinalIgnoreCase)) + _ = Task.Run(async () => + { + try + { + var body = await r.TextAsync(); + if (body.Contains("\"FormSubmitSuccess\"", StringComparison.Ordinal)) boardAccepted = true; + var err = Regex.Match(body, "\"errors\"\\s*:\\s*\\[\\s*\\{[^\\]]*?\"message\"\\s*:\\s*\"([^\"]{1,200})\""); + if (err.Success) boardSaid = err.Groups[1].Value; + } + catch (PlaywrightException) { /* body gone with the page */ } + }); // The board's own verdict on the application. Greenhouse answers a submission // its invisible reCAPTCHA Enterprise scored as a bot's with // 428 {"code":"captcha-failed", "security_code_recipient": ""} and asks @@ -555,7 +620,7 @@ async Task WaitForVerdictAsync(bool stopAtCodePrompt) await page.WaitForTimeoutAsync(1000); text = await BodyTextAsync(form); m = Confirmation().Match(text); - if (m.Success) break; + if (m.Success || boardAccepted) break; // "Are you sure you want to apply?" — a board that asks once more before it // sends. Answered once, and only inside a dialog the click opened (#216). if (!confirmedDialog && i >= 1 && await ConfirmDialogAsync(page)) @@ -603,6 +668,9 @@ string PostNote() { lock (posts) return posts.Count == 0 ? " (no application request was sent)" : $" ({string.Join("; ", posts)})"; } + // The page never said so, but the board's own answer did (#421). + if (!m.Success && boardAccepted) + return new SubmitOutcome(true, true, page.Url, "the board answered FormSubmitSuccess" + PostNote(), screenshot, unmapped, mapped, ""); if (!m.Success) { // A challenge thrown up BY the click is the common case, not the rare one: the @@ -639,8 +707,21 @@ string PostNote() if (await FormResetAsync(form, packet, mapped)) return new SubmitOutcome(true, false, page.Url, "", screenshot, unmapped, mapped, "Submit was clicked and the form reset itself with no confirmation" + PostNote() + " — check the screenshot"); + // The click sent nothing at all, and the form carries a captcha — the invisible + // kind included. Lever's Submit only asks hCaptcha for a token and posts the form + // from its callback; from this browser hCaptcha never answers, so the click is + // swallowed and nothing leaves. Both Lever runs read "no confirmation" for an + // application that was never posted (#420). That is a captcha holding the form, + // the person's to finish, not a mystery to retry. + bool noneSent; + lock (posts) noneSent = posts.Count == 0; + if (noneSent && page.Url == urlBefore && (await HasCaptchaWidgetAsync(form) || await HasCaptchaWidgetAsync(page.MainFrame))) + return new SubmitOutcome(true, false, page.Url, "", screenshot, unmapped, mapped, + "Submit sent nothing — the form's captcha held it (no application request was sent); finish it with Copy answers and open", + Captcha: true); return new SubmitOutcome(true, false, page.Url, "", screenshot, unmapped, mapped, - "Submit was clicked but no confirmation text was recognised" + PostNote() + " — check the screenshot"); + "Submit was clicked but no confirmation text was recognised" + PostNote() + + (boardSaid.Length > 0 ? $" — the board said \"{boardSaid}\"" : "") + " — check the screenshot"); } var start = Math.Max(0, m.Index - 80); var snippet = text.Substring(start, Math.Min(text.Length - start, 240)).Trim(); @@ -790,11 +871,62 @@ private static async Task FillAsync(IFrame page, PacketQuestion q, string if (DateInputValue(answer, inputType) is not { } iso) return false; answer = iso; } + if (inputType == "tel" || PhoneQuestion().IsMatch(Unprefixed(q.Id) + " " + q.Label)) + return await FillPhoneAsync(page, control, answer); try { await control.FillAsync(answer); } catch (PlaywrightException) { return false; } return true; } + [GeneratedRegex(@"\bphone\b|\bmobile\b|\bcell\b|\btelephone\b", RegexOptions.IgnoreCase)] + private static partial Regex PhoneQuestion(); + + /// + /// Type a phone number the way its box takes it. An international widget shows the country's + /// dial code in the box before anything is typed — Motion Recruitment's react-international- + /// phone holds "+1 " — and resets a fill() wholesale: the box kept "+1 ", read as filled, and + /// the board said "This phone is invalid or does not exist" (#438). There the national number + /// is typed after the code, a key at a time, as a person would. Anywhere else the answer is + /// filled as it is, and typed instead if the box did not keep its digits. + /// + private static async Task FillPhoneAsync(IFrame page, ILocator control, string answer) + { + try + { + var before = await control.InputValueAsync(new() { Timeout = 2_000 }); + var prefix = DialCodePrefix(before); + var national = prefix is null ? Digits(answer) : NationalDigits(answer, prefix); + if (prefix is null) + { + await control.FillAsync(answer); + if (Digits(await control.InputValueAsync()).EndsWith(Digits(answer), StringComparison.Ordinal)) return true; + await control.FillAsync(""); + } + await control.FocusAsync(); + await page.Page.Keyboard.PressAsync("End"); + await control.PressSequentiallyAsync(prefix is null ? answer : national, new() { Delay = 30 }); + return Digits(await control.InputValueAsync()).EndsWith(national, StringComparison.Ordinal); + } + catch (Exception ex) when (ex is PlaywrightException or TimeoutException) { return false; } + } + + private static string Digits(string s) => new([.. s.Where(char.IsAsciiDigit)]); + + /// The dial code a phone box shows before anything is typed ("+1 " → "1"), or null + /// when the box holds anything else. Public for tests. + public static string? DialCodePrefix(string value) => + Regex.Match(value ?? "", @"^\s*\+\s*(\d{1,4})\s*$") is { Success: true } m ? m.Groups[1].Value : null; + + /// The number to type after a widget's dial code: the answer's digits, without the + /// code when the answer carries it ("+1 571 297 5406" or "15712975406" after "+1" → + /// "5712975406"). Public for tests. + public static string NationalDigits(string answer, string dialCode) + { + var digits = Digits(answer); + var international = answer.TrimStart().StartsWith('+') || (dialCode == "1" && digits.Length == 11); + return international && digits.StartsWith(dialCode, StringComparison.Ordinal) ? digits[dialCode.Length..] : digits; + } + /// An answer in the form a native date, month or datetime-local input accepts, or /// null when it names no date: "October 1, 2026" → "2026-10-01" / "2026-10" / "2026-10-01T09:00". public static string? DateInputValue(string answer, string inputType) @@ -822,8 +954,78 @@ private static async Task FillAsync(IFrame page, PacketQuestion q, string private static async Task ChooseAsync(IFrame page, ILocator control, PacketQuestion q, string answer) { var fixedSet = q.Options.Count > 0; - if (fixedSet && !q.Options.Any(o => Same(o, answer))) - return false; + if (!fixedSet) return await ChooseOneAsync(page, control, q, answer, fixedSet: false); + // A fixed list takes only its own options — the answer as given, or the options it means + // ("United States of America" is the list's "US", a multi-select's "AWS, Azure" is two + // picks, #432). Each is chosen in turn; a multi widget keeps the earlier ones. + var multi = q.Type == PacketQuestion.MultiSelect || await IsMultiWidgetAsync(control); + var picks = q.Options.FirstOrDefault(o => Same(o, answer)) is { } exact ? [exact] : MenuChoices(q.Options, answer, multi, q.Label); + if (picks.Count == 0) return false; + foreach (var pick in picks) + if (!await ChooseOneAsync(page, control, q, pick, fixedSet: true)) return false; + return true; + } + + /// + /// The options of a menu an answer means, when it is not one of them word for word — the + /// drafter wrote to a question it could not see the choices of (#432). In order: the options + /// a comma-separated answer lists; the country the answer names under any of its aliases; + /// on a Yes/No menu, the yes or no the answer opens with — or, for a short answer with no + /// negative in it ("US Citizen" to "Are you legally authorized…?"), Yes, never on a + /// sponsorship or visa question, whose yes is the negative one; else the options the answer + /// names outright ("…plus Rust…" on Ruby on Rails / Rust / Both / Neither), one for a single + /// choice and only when it is the only one named. Empty when nothing reads as meant. Public + /// for tests. + /// + public static List MenuChoices(IReadOnlyList options, string answer, bool multi, string label = "") + { + var real = options.Select(o => o.Trim()).Where(o => o.Length > 0 && !Placeholder().IsMatch(o)).ToList(); + var a = answer.Trim(); + if (real.Count == 0 || a.Length == 0) return []; + if (real.FirstOrDefault(o => Same(o, a)) is { } exact) return [exact]; + var parts = a.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + if (multi && parts.Length > 1) + { + var listed = parts.Select(p => real.FirstOrDefault(o => Same(o, p))).ToList(); + if (listed.All(o => o is not null)) return listed.Select(o => o!).Distinct().ToList(); + } + if (AnswerDrafter.PickCountryOption(real, AnswerDrafter.CanonicalCountry(a)) is { } country) return [country]; + var yes = real.FirstOrDefault(o => Regex.IsMatch(o, @"^\s*yes\b", RegexOptions.IgnoreCase)); + var no = real.FirstOrDefault(o => Regex.IsMatch(o, @"^\s*no\b", RegexOptions.IgnoreCase)); + if (yes is not null && no is not null && real.Count <= 3) + { + var lead = Regex.Match(a, @"^\W*(yes|no)\b", RegexOptions.IgnoreCase); + if (lead.Success) return [lead.Groups[1].Value.Equals("yes", StringComparison.OrdinalIgnoreCase) ? yes : no]; + var shortAnswer = a.Split(' ', StringSplitOptions.RemoveEmptyEntries).Length <= 4; + if (shortAnswer && !Regex.IsMatch(label, @"sponsor|visa", RegexOptions.IgnoreCase) + && !Regex.IsMatch(a, @"\b(?:not|no|never|none|n['’]t|require[sd]?|need(?:s|ed)?)\b", RegexOptions.IgnoreCase)) + return [yes]; + return []; + } + // Named, and not denied: "No Ruby on Rails experience" names Ruby on Rails to say no to it. + var named = real.Where(o => o.Length >= 2 && Regex.Matches(a, @"(? !Regex.IsMatch(a[Math.Max(0, m.Index - 24)..m.Index].Split('.', ';', ',', '—', '(').Last(), + @"\b(?:no|not|without|never|lack|nor|neither)\b", RegexOptions.IgnoreCase))).ToList(); + return multi ? named : named.Count == 1 ? named : []; + } + + /// A combobox that takes several values: react-select marks its value container + /// --is-multi. + private static async Task IsMultiWidgetAsync(ILocator control) + { + try + { + return await control.EvaluateAsync(""" + el => { for (let a = el.parentElement, i = 0; a && i < 4; a = a.parentElement, i++) + if (a.matches('[class*="is-multi"]')) return true; + return false; } + """); + } + catch (PlaywrightException) { return false; } + } + + private static async Task ChooseOneAsync(IFrame page, ILocator control, PacketQuestion q, string answer, bool fixedSet) + { var combobox = await IsComboboxAsync(control); ILocator? pick = null; var typed = answer; @@ -856,6 +1058,36 @@ private static async Task ChooseAsync(IFrame page, ILocator control, Packe } if (pick is not null) break; } + // Typed, the answer filtered the menu down to nothing: it is prose written to a question + // whose choices the drafter never saw ("Yes. At Ronin 48 I…" to a Yes/No, #432). Empty the + // box, read the whole menu, and choose what the answer means — or nothing. + if (pick is null && combobox && !elsewhere && !fixedSet) + { + try + { + await control.FillAsync("", new() { Timeout = 3_000 }); + var options = page.Locator("[role=option]:visible"); + IReadOnlyList texts = []; + for (var i = 0; i < 8 && texts.Count == 0; i++) + { + await page.WaitForTimeoutAsync(250); + texts = await options.AllInnerTextsAsync(); + } + var meant = MenuChoices([.. texts.Select(t => t.Replace('\n', ' ').Trim())], answer, + q.Type == PacketQuestion.MultiSelect || await IsMultiWidgetAsync(control), q.Label); + if (meant.Count > 0) + { + await page.Page.Keyboard.PressAsync("Escape"); + foreach (var choice in meant) + if (!await ChooseOneAsync(page, control, q, choice, fixedSet: true)) return false; + return true; + } + // Nothing meant: put the answer back, or the Enter below would commit whichever + // option the emptied menu has first. + await control.FillAsync(typed, new() { Timeout = 3_000 }); + } + catch (Exception ex) when (ex is PlaywrightException or TimeoutException) { /* fall through to the old commit */ } + } // No menu ever showed: commit whatever is under the caret. Only on a combobox — // Enter in a plain text input submits the form it sits in — and never in a JET filter // box, which commits nothing but a chosen row and is a plain input to its form. @@ -864,7 +1096,26 @@ private static async Task ChooseAsync(IFrame page, ILocator control, Packe if (pick is not null) { try { await pick.ClickAsync(new() { Timeout = 5_000 }); } - catch (Exception ex) when (ex is PlaywrightException or TimeoutException) { pick = null; } + catch (Exception ex) when (ex is PlaywrightException or TimeoutException) + { + pick = null; + // A notice the typing brought up took the click — Flexhire opens its location + // notice as Location is typed (#437). Acknowledged, the suggestion is sought again. + if (combobox && !elsewhere && await BrowserSession.AcknowledgeNoticeAsync(page.Page)) + { + try + { + await control.FillAsync(typed, new() { Timeout = 3_000 }); + for (var i = 0; i < 12 && pick is null; i++) + { + await page.WaitForTimeoutAsync(250); + pick = await BestOptionAsync(page, control, answer, fixedSet); + } + if (pick is not null) await pick.ClickAsync(new() { Timeout = 5_000 }); + } + catch (Exception again) when (again is PlaywrightException or TimeoutException) { pick = null; } + } + } } // A JET dropdown left open with nothing chosen (a dependent list whose parent is still // empty) is closed, or its popup sits over the next field and its own box stays hidden. @@ -1010,14 +1261,19 @@ private static async Task CommittedAsync(ILocator control, string chosen) private static readonly string RequiredEmptyScript = """ () => { const widget = __WIDGET__; + // Shadow roots included, and a label looked up in its control's own root (#436). + const deep = __DEEP__; + const rootOf = el => { const r = el.getRootNode(); return r && r !== document && r.querySelector ? r : document; }; const visible = el => { const r = el.getBoundingClientRect(); const s = getComputedStyle(el); return r.width > 0 && r.height > 0 && s.visibility !== 'hidden' && s.display !== 'none'; }; const labelFor = el => { if (el.getAttribute('aria-label')) return el.getAttribute('aria-label'); const by = el.getAttribute('aria-labelledby'); - if (by) { const t = by.split(/\s+/).map(i => document.getElementById(i)?.innerText || '').join(' ').trim(); if (t) return t; } - if (el.id) { const l = document.querySelector(`label[for="${CSS.escape(el.id)}"]`); if (l) return l.innerText; } - const wrap = el.closest('label'); if (wrap) return wrap.innerText; + if (by) { const t = by.split(/\s+/).map(i => (rootOf(el).getElementById ? rootOf(el).getElementById(i) : null)?.innerText || document.getElementById(i)?.innerText || '').join(' ').trim(); if (t) return t; } + if (el.id) { const l = rootOf(el).querySelector(`label[for="${CSS.escape(el.id)}"]`); if (l) return l.innerText; } + // A dropzone's instructions are not the field's name; the row's title is (#417). + const wrap = el.closest('label'); + if (wrap && !(el.type === 'file' && (__DROPZONE__)(wrap.innerText || ''))) return wrap.innerText; const legend = el.closest('fieldset')?.querySelector('legend'); if (legend) return legend.innerText; // The row's own label when the page never tied it to the control — the same rule // discovery names the field by, so the sweep and the packet agree on it. SuccessFactors' @@ -1054,7 +1310,7 @@ private static async Task CommittedAsync(ILocator control, string chosen) // complained, when the field's own Enter manually box has text in it, or when the // widget is showing a file name. Judged over the whole field, not the input: an // uploader's error and its text twin both live beside the input, not on it. - const uploadError = /cannot read propert|uploadfile|upload failed|failed to upload|error uploading/i; + const uploadError = /cannot read propert|uploadfile|upload failed|failed to upload|error uploading|unable to upload/i; const fileFilled = root => { const text = root.innerText || ''; if (uploadError.test(text)) return false; @@ -1111,11 +1367,19 @@ private static async Task CommittedAsync(ILocator control, string chosen) // the radios' name, whose first half is this page load's and no other's. if (empty) add(title.getAttribute('for') || choices[0]?.getAttribute('name') || '', title.innerText); } - for (const el of document.querySelectorAll('input, select, textarea')) { + for (const el of deep('input, select, textarea')) { if (widget(el)) continue; // a search or job-alert box is never a required field (#214) const type = (el.getAttribute('type') || (el.tagName === 'SELECT' ? 'select' : 'text')).toLowerCase(); if (['hidden', 'submit', 'button', 'reset', 'image'].includes(type) || el.disabled) continue; - if (!(el.required || el.getAttribute('aria-required') === 'true')) continue; + // Required by the star in its label alone, the way discovery reads it: HubSpot's own + // rendering of its Greenhouse form marks a dozen selects "… *" and nothing else, the + // sweep let them all through blank, and Submit stayed disabled — "no Submit button + // found" (#425); Manatal marks every required box only that way too (#436). Boxes, + // selects and textareas only, on screen: a choice group's star is on its title, which + // discovery and the radio rule below already read. + const starred = !['radio', 'checkbox', 'file'].includes(type) && visible(el) && !el.readOnly + && (el.id || el.getAttribute('name')) && /^\s*\*|\*\s*$/.test((labelFor(el) || '').trim()); + if (!(el.required || el.getAttribute('aria-required') === 'true' || starred)) continue; if (sideForm(el)) continue; if (type === 'file') { // Not gated on the input being visible: uploaders keep theirs off screen behind @@ -1147,13 +1411,13 @@ private static async Task CommittedAsync(ILocator control, string chosen) // box by box, every unticked one read as a required field left empty, and Fieldwire's // fully answered form was refused the click (#280). const name = el.getAttribute('name') || ''; - const group = name ? [...document.querySelectorAll(`input[type=checkbox][name="${CSS.escape(name)}"]`)] : [el]; + const group = name ? [...rootOf(el).querySelectorAll(`input[type=checkbox][name="${CSS.escape(name)}"]`)] : [el]; empty = !group.some(c => c.checked); if (empty && group.length > 1) { add(name, labelFor(el)); continue; } } else if (type === 'radio') { const name = el.getAttribute('name') || ''; - empty = ![...document.querySelectorAll(`input[type=radio][name="${CSS.escape(name)}"]`)].some(r => r.checked); + empty = ![...rootOf(el).querySelectorAll(`input[type=radio][name="${CSS.escape(name)}"]`)].some(r => r.checked); // Once per group, by its name and the question over it — never by an option (#318). if (empty && drawnRadio) { const g = el.closest('[role=radiogroup]'); @@ -1162,6 +1426,11 @@ private static async Task CommittedAsync(ILocator control, string chosen) continue; } } + // A select still on its placeholder is empty even when the placeholder carries a value: + // HubSpot's are (#425). + else if (el.tagName === 'SELECT') + empty = (el.value || '').trim().length === 0 + || (el.selectedIndex === 0 && /^\s*-*\s*(?:select|choose|please (?:select|choose)|pick)\b/i.test(el.options[0]?.text || '')); else empty = (el.value || '').trim().length === 0; if (empty) add(el.id || el.getAttribute('name'), labelFor(el)); } @@ -1188,7 +1457,7 @@ private static async Task CommittedAsync(ILocator control, string chosen) } return out; } - """.Replace("__WIDGET__", BrowserSession.WidgetJs); + """.Replace("__WIDGET__", BrowserSession.WidgetJs).Replace("__DROPZONE__", BrowserSession.DropzoneJs).Replace("__DEEP__", BrowserSession.DeepJs); /// Greenhouse's security-code prompt: one box per character, ids security-input-N, /// under "A verification code was sent to …". @@ -1308,7 +1577,7 @@ public static string CandidateEmail(AgentPacket packet) if (email.Length == 0) return Stop(SignInError + ", and the packet has no email address to sign in with"); - var box = form.Locator("input[type=email]:visible").First; + var box = form.Locator(EmailBox).First; var textBefore = await BodyTextAsync(form); var urlBefore = page.Url; try @@ -1394,10 +1663,15 @@ private static async Task AfterContinueAsync(IPage page, string text } } + /// A sign-in's email box: type=email, or a text box that names itself one (#429). + private const string EmailBox = + "input[type=email]:visible, input[type=text][autocomplete=email]:visible, input[type=text][name*=email i]:visible, " + + "input[type=text][id*=email i]:visible, input[type=text][placeholder*=email i]:visible, input:not([type])[placeholder*=email i]:visible"; + /// Is the sign-in's own email box still on screen? False mid-navigation too. private static async Task EmailBoxShownAsync(IPage page) { - try { return await page.MainFrame.Locator("input[type=email]:visible").First.IsVisibleAsync(); } + try { return await page.MainFrame.Locator(EmailBox).First.IsVisibleAsync(); } catch (PlaywrightException) { return false; } } @@ -1508,6 +1782,8 @@ private static async Task SubmitOnPageAsync(IFrame page) page.Locator("#submit_app, [id$='_submitBtn']").Filter(notLater).First, page.GetByRole(AriaRole.Button, new() { NameRegex = new Regex(@"^submit\b|submit (?:my |your |the )?application", RegexOptions.IgnoreCase) }).Filter(notLater).First, page.Locator("button[type=submit], input[type=submit]").Filter(new() { HasTextRegex = new Regex(@"^\s*(?:submit|apply)\b", RegexOptions.IgnoreCase) }).Filter(notLater).First, + // A script's anchor that submits the form (#424). + page.Locator("form").Locator(FormAnchorSubmit).Filter(new() { HasTextRegex = AnchorSubmitWords() }).Filter(notLater).First, }) { try { if (await c.CountAsync() > 0 && await c.IsVisibleAsync()) return true; } @@ -1541,6 +1817,16 @@ private async Task